2024-12-10 06:14:02,549 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 06:14:02,561 main DEBUG Took 0.009920 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 06:14:02,561 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 06:14:02,562 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 06:14:02,562 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 06:14:02,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,570 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 06:14:02,581 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,583 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,584 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,585 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,586 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,587 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,588 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,588 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,589 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,590 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,590 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,591 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:14:02,591 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,592 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 06:14:02,593 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:14:02,594 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 06:14:02,596 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 06:14:02,596 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 06:14:02,597 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 06:14:02,598 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 06:14:02,606 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 06:14:02,608 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 06:14:02,610 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 06:14:02,611 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 06:14:02,611 main DEBUG createAppenders(={Console}) 2024-12-10 06:14:02,612 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-10 06:14:02,612 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 06:14:02,612 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-10 06:14:02,613 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 06:14:02,613 main DEBUG OutputStream closed 2024-12-10 06:14:02,613 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 06:14:02,613 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 06:14:02,614 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-10 06:14:02,696 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 06:14:02,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 06:14:02,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 06:14:02,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 06:14:02,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 06:14:02,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 06:14:02,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 06:14:02,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 06:14:02,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 06:14:02,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 06:14:02,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 06:14:02,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 06:14:02,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 06:14:02,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 06:14:02,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 06:14:02,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 06:14:02,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 06:14:02,709 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 06:14:02,712 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 06:14:02,713 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-10 06:14:02,713 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 06:14:02,714 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-10T06:14:03,031 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d 2024-12-10 06:14:03,034 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 06:14:03,034 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T06:14:03,043 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-10T06:14:03,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-10T06:14:03,077 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982, deleteOnExit=true 2024-12-10T06:14:03,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-10T06:14:03,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/test.cache.data in system properties and HBase conf 2024-12-10T06:14:03,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/hadoop.tmp.dir in system properties and HBase conf 2024-12-10T06:14:03,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/hadoop.log.dir in system properties and HBase conf 2024-12-10T06:14:03,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-10T06:14:03,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-10T06:14:03,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-10T06:14:03,171 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T06:14:03,275 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T06:14:03,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-10T06:14:03,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-10T06:14:03,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-10T06:14:03,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T06:14:03,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-10T06:14:03,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-10T06:14:03,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T06:14:03,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T06:14:03,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-10T06:14:03,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/nfs.dump.dir in system properties and HBase conf 2024-12-10T06:14:03,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/java.io.tmpdir in system properties and HBase conf 2024-12-10T06:14:03,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T06:14:03,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-10T06:14:03,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-10T06:14:04,146 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-10T06:14:04,242 INFO [Time-limited test {}] log.Log(170): Logging initialized @2424ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-10T06:14:04,325 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T06:14:04,391 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T06:14:04,412 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T06:14:04,412 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T06:14:04,414 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T06:14:04,430 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T06:14:04,433 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/hadoop.log.dir/,AVAILABLE} 2024-12-10T06:14:04,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T06:14:04,626 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/java.io.tmpdir/jetty-localhost-43331-hadoop-hdfs-3_4_1-tests_jar-_-any-12874104370073941156/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T06:14:04,633 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43331} 2024-12-10T06:14:04,634 INFO [Time-limited test {}] server.Server(415): Started @2816ms 2024-12-10T06:14:05,039 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T06:14:05,046 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T06:14:05,047 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T06:14:05,047 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T06:14:05,048 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T06:14:05,048 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/hadoop.log.dir/,AVAILABLE} 2024-12-10T06:14:05,049 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T06:14:05,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/java.io.tmpdir/jetty-localhost-40003-hadoop-hdfs-3_4_1-tests_jar-_-any-13587756778509846667/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T06:14:05,170 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40003} 2024-12-10T06:14:05,170 INFO [Time-limited test {}] server.Server(415): Started @3353ms 2024-12-10T06:14:05,227 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T06:14:05,749 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982/data/data2/current/BP-959695638-172.17.0.2-1733811243885/current, will proceed with Du for space computation calculation, 2024-12-10T06:14:05,749 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982/data/data1/current/BP-959695638-172.17.0.2-1733811243885/current, will proceed with Du for space computation calculation, 2024-12-10T06:14:05,788 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T06:14:05,860 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa176a2452682aa05 with lease ID 0x7380dad16b0a92b1: Processing first storage report for DS-5636f389-6925-4170-b0d5-678af6414ef2 from datanode DatanodeRegistration(127.0.0.1:35903, datanodeUuid=1053313a-5eb1-45b0-8331-7fb75bfc6be1, infoPort=45429, infoSecurePort=0, ipcPort=36349, storageInfo=lv=-57;cid=testClusterID;nsid=972669145;c=1733811243885) 2024-12-10T06:14:05,863 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa176a2452682aa05 with lease ID 0x7380dad16b0a92b1: from storage DS-5636f389-6925-4170-b0d5-678af6414ef2 node DatanodeRegistration(127.0.0.1:35903, datanodeUuid=1053313a-5eb1-45b0-8331-7fb75bfc6be1, infoPort=45429, infoSecurePort=0, ipcPort=36349, storageInfo=lv=-57;cid=testClusterID;nsid=972669145;c=1733811243885), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-12-10T06:14:05,864 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa176a2452682aa05 with lease ID 0x7380dad16b0a92b1: Processing first storage report for DS-cce30ae5-8b15-414d-ba88-5dfe05a96b7b from datanode DatanodeRegistration(127.0.0.1:35903, datanodeUuid=1053313a-5eb1-45b0-8331-7fb75bfc6be1, infoPort=45429, infoSecurePort=0, ipcPort=36349, storageInfo=lv=-57;cid=testClusterID;nsid=972669145;c=1733811243885) 2024-12-10T06:14:05,864 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa176a2452682aa05 with lease ID 0x7380dad16b0a92b1: from storage DS-cce30ae5-8b15-414d-ba88-5dfe05a96b7b node DatanodeRegistration(127.0.0.1:35903, datanodeUuid=1053313a-5eb1-45b0-8331-7fb75bfc6be1, infoPort=45429, infoSecurePort=0, ipcPort=36349, storageInfo=lv=-57;cid=testClusterID;nsid=972669145;c=1733811243885), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T06:14:05,919 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d 2024-12-10T06:14:05,998 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982/zookeeper_0, clientPort=55003, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T06:14:06,009 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55003 2024-12-10T06:14:06,026 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:14:06,030 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:14:06,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741825_1001 (size=7) 2024-12-10T06:14:06,680 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6 with version=8 2024-12-10T06:14:06,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/hbase-staging 2024-12-10T06:14:06,779 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T06:14:07,052 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c42c682b34f7:0 server-side Connection retries=45 2024-12-10T06:14:07,064 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:14:07,064 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T06:14:07,070 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T06:14:07,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:14:07,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T06:14:07,221 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T06:14:07,284 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T06:14:07,292 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T06:14:07,296 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T06:14:07,324 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3050 (auto-detected) 2024-12-10T06:14:07,325 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-10T06:14:07,345 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37415 2024-12-10T06:14:07,368 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37415 connecting to ZooKeeper ensemble=127.0.0.1:55003 2024-12-10T06:14:07,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:374150x0, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T06:14:07,412 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37415-0x1019b01fe2c0000 connected 2024-12-10T06:14:07,446 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:14:07,449 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:14:07,462 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:14:07,467 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6, hbase.cluster.distributed=false 2024-12-10T06:14:07,494 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T06:14:07,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37415 2024-12-10T06:14:07,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37415 2024-12-10T06:14:07,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37415 2024-12-10T06:14:07,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37415 2024-12-10T06:14:07,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37415 2024-12-10T06:14:07,624 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c42c682b34f7:0 server-side Connection retries=45 2024-12-10T06:14:07,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:14:07,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T06:14:07,627 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T06:14:07,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:14:07,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T06:14:07,630 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T06:14:07,632 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T06:14:07,633 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43983 2024-12-10T06:14:07,635 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43983 connecting to ZooKeeper ensemble=127.0.0.1:55003 2024-12-10T06:14:07,637 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:14:07,641 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:14:07,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:439830x0, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T06:14:07,651 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43983-0x1019b01fe2c0001 connected 2024-12-10T06:14:07,651 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:14:07,657 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T06:14:07,666 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T06:14:07,669 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T06:14:07,674 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T06:14:07,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43983 2024-12-10T06:14:07,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43983 2024-12-10T06:14:07,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43983 2024-12-10T06:14:07,676 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43983 2024-12-10T06:14:07,676 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43983 2024-12-10T06:14:07,692 DEBUG [M:0;c42c682b34f7:37415 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c42c682b34f7:37415 2024-12-10T06:14:07,693 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c42c682b34f7,37415,1733811246834 2024-12-10T06:14:07,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:14:07,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:14:07,702 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c42c682b34f7,37415,1733811246834 2024-12-10T06:14:07,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:07,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T06:14:07,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:07,723 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T06:14:07,725 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c42c682b34f7,37415,1733811246834 from backup master directory 2024-12-10T06:14:07,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c42c682b34f7,37415,1733811246834 2024-12-10T06:14:07,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:14:07,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:14:07,729 WARN [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T06:14:07,729 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c42c682b34f7,37415,1733811246834 2024-12-10T06:14:07,732 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T06:14:07,733 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T06:14:07,790 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/hbase.id] with ID: fa859e13-0c00-4715-a487-f5a4e3af8477 2024-12-10T06:14:07,790 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/.tmp/hbase.id 2024-12-10T06:14:07,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741826_1002 (size=42) 2024-12-10T06:14:08,202 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/.tmp/hbase.id]:[hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/hbase.id] 2024-12-10T06:14:08,249 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:14:08,256 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T06:14:08,279 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-12-10T06:14:08,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:08,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:08,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741827_1003 (size=196) 2024-12-10T06:14:08,717 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:14:08,720 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T06:14:08,738 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:08,742 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:14:08,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741828_1004 (size=1189) 2024-12-10T06:14:09,197 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store 2024-12-10T06:14:09,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741829_1005 (size=34) 2024-12-10T06:14:09,628 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T06:14:09,633 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:14:09,635 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T06:14:09,635 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:14:09,636 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:14:09,640 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T06:14:09,641 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:14:09,642 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:14:09,652 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733811249635Disabling compacts and flushes for region at 1733811249635Disabling writes for close at 1733811249640 (+5 ms)Writing region close event to WAL at 1733811249642 (+2 ms)Closed at 1733811249642 2024-12-10T06:14:09,657 WARN [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/.initializing 2024-12-10T06:14:09,657 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/WALs/c42c682b34f7,37415,1733811246834 2024-12-10T06:14:09,670 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:14:09,697 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c42c682b34f7%2C37415%2C1733811246834, suffix=, logDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/WALs/c42c682b34f7,37415,1733811246834, archiveDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/oldWALs, maxLogs=10 2024-12-10T06:14:09,735 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/WALs/c42c682b34f7,37415,1733811246834/c42c682b34f7%2C37415%2C1733811246834.1733811249704, exclude list is [], retry=0 2024-12-10T06:14:09,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35903,DS-5636f389-6925-4170-b0d5-678af6414ef2,DISK] 2024-12-10T06:14:09,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-10T06:14:09,812 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/WALs/c42c682b34f7,37415,1733811246834/c42c682b34f7%2C37415%2C1733811246834.1733811249704 2024-12-10T06:14:09,813 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45429:45429)] 2024-12-10T06:14:09,814 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:14:09,814 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:14:09,818 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,820 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T06:14:09,903 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:09,906 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:14:09,907 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,911 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T06:14:09,911 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:09,912 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:09,913 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T06:14:09,916 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:09,917 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:09,917 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T06:14:09,921 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:09,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:09,922 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,927 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,928 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,934 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,935 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,939 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T06:14:09,943 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:14:09,947 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:14:09,948 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67367125, jitterRate=0.0038483887910842896}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T06:14:09,953 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733811249836Initializing all the Stores at 1733811249839 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733811249840 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811249844 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811249845 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811249845Cleaning up temporary data from old regions at 1733811249936 (+91 ms)Region opened successfully at 1733811249953 (+17 ms) 2024-12-10T06:14:09,955 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T06:14:09,988 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2972663e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c42c682b34f7/172.17.0.2:0 2024-12-10T06:14:10,022 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T06:14:10,034 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T06:14:10,035 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T06:14:10,038 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T06:14:10,039 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-10T06:14:10,045 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-10T06:14:10,045 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T06:14:10,078 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T06:14:10,090 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T06:14:10,093 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T06:14:10,096 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T06:14:10,098 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T06:14:10,101 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T06:14:10,103 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T06:14:10,108 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T06:14:10,110 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T06:14:10,112 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T06:14:10,116 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T06:14:10,135 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T06:14:10,137 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T06:14:10,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T06:14:10,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T06:14:10,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:10,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:10,144 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c42c682b34f7,37415,1733811246834, sessionid=0x1019b01fe2c0000, setting cluster-up flag (Was=false) 2024-12-10T06:14:10,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:10,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:10,163 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T06:14:10,165 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c42c682b34f7,37415,1733811246834 2024-12-10T06:14:10,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:10,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:10,177 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T06:14:10,179 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c42c682b34f7,37415,1733811246834 2024-12-10T06:14:10,185 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T06:14:10,259 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T06:14:10,269 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T06:14:10,275 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T06:14:10,283 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(746): ClusterId : fa859e13-0c00-4715-a487-f5a4e3af8477 2024-12-10T06:14:10,282 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c42c682b34f7,37415,1733811246834 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T06:14:10,286 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T06:14:10,291 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T06:14:10,291 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T06:14:10,292 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c42c682b34f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T06:14:10,292 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c42c682b34f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T06:14:10,292 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c42c682b34f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T06:14:10,292 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c42c682b34f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T06:14:10,292 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c42c682b34f7:0, corePoolSize=10, maxPoolSize=10 2024-12-10T06:14:10,293 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,293 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c42c682b34f7:0, corePoolSize=2, maxPoolSize=2 2024-12-10T06:14:10,293 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,295 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T06:14:10,295 DEBUG [RS:0;c42c682b34f7:43983 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33894e69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c42c682b34f7/172.17.0.2:0 2024-12-10T06:14:10,300 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T06:14:10,300 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T06:14:10,307 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:10,307 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T06:14:10,315 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c42c682b34f7:43983 2024-12-10T06:14:10,319 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T06:14:10,320 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T06:14:10,320 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T06:14:10,323 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(2659): reportForDuty to master=c42c682b34f7,37415,1733811246834 with port=43983, startcode=1733811247584 2024-12-10T06:14:10,327 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733811280327 2024-12-10T06:14:10,329 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T06:14:10,330 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T06:14:10,334 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T06:14:10,335 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T06:14:10,335 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T06:14:10,335 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T06:14:10,339 DEBUG [RS:0;c42c682b34f7:43983 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T06:14:10,345 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741831_1007 (size=1321) 2024-12-10T06:14:10,351 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T06:14:10,352 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6 2024-12-10T06:14:10,353 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T06:14:10,355 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T06:14:10,356 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T06:14:10,360 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T06:14:10,361 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T06:14:10,368 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.large.0-1733811250362,5,FailOnTimeoutGroup] 2024-12-10T06:14:10,372 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.small.0-1733811250368,5,FailOnTimeoutGroup] 2024-12-10T06:14:10,372 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,373 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T06:14:10,374 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,374 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741832_1008 (size=32) 2024-12-10T06:14:10,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:14:10,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T06:14:10,390 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T06:14:10,390 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:10,391 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32795, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T06:14:10,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:14:10,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T06:14:10,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T06:14:10,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:10,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:14:10,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T06:14:10,398 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37415 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c42c682b34f7,43983,1733811247584 2024-12-10T06:14:10,400 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T06:14:10,401 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:10,402 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37415 {}] master.ServerManager(517): Registering regionserver=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:10,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:14:10,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T06:14:10,405 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T06:14:10,405 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:10,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:14:10,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T06:14:10,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740 2024-12-10T06:14:10,412 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740 2024-12-10T06:14:10,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T06:14:10,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T06:14:10,418 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:14:10,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T06:14:10,422 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6 2024-12-10T06:14:10,422 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40237 2024-12-10T06:14:10,422 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T06:14:10,428 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:14:10,430 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61544690, jitterRate=-0.08291265368461609}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:14:10,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T06:14:10,433 DEBUG [RS:0;c42c682b34f7:43983 {}] zookeeper.ZKUtil(111): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c42c682b34f7,43983,1733811247584 2024-12-10T06:14:10,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733811250383Initializing all the Stores at 1733811250385 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733811250385Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733811250386 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811250386Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733811250386Cleaning up temporary data from old regions at 1733811250416 (+30 ms)Region opened successfully at 1733811250433 (+17 ms) 2024-12-10T06:14:10,433 WARN [RS:0;c42c682b34f7:43983 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T06:14:10,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T06:14:10,433 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T06:14:10,433 INFO [RS:0;c42c682b34f7:43983 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:14:10,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T06:14:10,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T06:14:10,434 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/WALs/c42c682b34f7,43983,1733811247584 2024-12-10T06:14:10,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T06:14:10,435 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c42c682b34f7,43983,1733811247584] 2024-12-10T06:14:10,437 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T06:14:10,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733811250433Disabling compacts and flushes for region at 1733811250433Disabling writes for close at 1733811250434 (+1 ms)Writing region close event to WAL at 1733811250437 (+3 ms)Closed at 1733811250437 2024-12-10T06:14:10,445 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T06:14:10,445 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T06:14:10,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T06:14:10,463 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T06:14:10,467 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T06:14:10,473 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T06:14:10,494 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T06:14:10,500 INFO [RS:0;c42c682b34f7:43983 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T06:14:10,500 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,502 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T06:14:10,512 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T06:14:10,514 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,514 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,515 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,515 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,515 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,515 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,515 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0, corePoolSize=2, maxPoolSize=2 2024-12-10T06:14:10,516 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,516 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,516 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,516 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,516 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,517 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:14:10,517 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c42c682b34f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T06:14:10,517 DEBUG [RS:0;c42c682b34f7:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T06:14:10,520 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,520 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,520 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,521 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,521 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,521 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,43983,1733811247584-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T06:14:10,554 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T06:14:10,557 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,43983,1733811247584-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,558 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,558 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.Replication(171): c42c682b34f7,43983,1733811247584 started 2024-12-10T06:14:10,586 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:10,587 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1482): Serving as c42c682b34f7,43983,1733811247584, RpcServer on c42c682b34f7/172.17.0.2:43983, sessionid=0x1019b01fe2c0001 2024-12-10T06:14:10,588 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T06:14:10,589 DEBUG [RS:0;c42c682b34f7:43983 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c42c682b34f7,43983,1733811247584 2024-12-10T06:14:10,589 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c42c682b34f7,43983,1733811247584' 2024-12-10T06:14:10,589 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T06:14:10,591 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T06:14:10,593 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T06:14:10,593 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T06:14:10,593 DEBUG [RS:0;c42c682b34f7:43983 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c42c682b34f7,43983,1733811247584 2024-12-10T06:14:10,593 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c42c682b34f7,43983,1733811247584' 2024-12-10T06:14:10,593 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T06:14:10,594 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T06:14:10,595 DEBUG [RS:0;c42c682b34f7:43983 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T06:14:10,595 INFO [RS:0;c42c682b34f7:43983 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T06:14:10,595 INFO [RS:0;c42c682b34f7:43983 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T06:14:10,619 WARN [c42c682b34f7:37415 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T06:14:10,702 INFO [RS:0;c42c682b34f7:43983 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:14:10,706 INFO [RS:0;c42c682b34f7:43983 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c42c682b34f7%2C43983%2C1733811247584, suffix=, logDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/WALs/c42c682b34f7,43983,1733811247584, archiveDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/oldWALs, maxLogs=32 2024-12-10T06:14:10,727 DEBUG [RS:0;c42c682b34f7:43983 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/WALs/c42c682b34f7,43983,1733811247584/c42c682b34f7%2C43983%2C1733811247584.1733811250709, exclude list is [], retry=0 2024-12-10T06:14:10,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35903,DS-5636f389-6925-4170-b0d5-678af6414ef2,DISK] 2024-12-10T06:14:10,738 INFO [RS:0;c42c682b34f7:43983 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/WALs/c42c682b34f7,43983,1733811247584/c42c682b34f7%2C43983%2C1733811247584.1733811250709 2024-12-10T06:14:10,738 DEBUG [RS:0;c42c682b34f7:43983 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45429:45429)] 2024-12-10T06:14:10,872 DEBUG [c42c682b34f7:37415 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T06:14:10,888 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:10,897 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c42c682b34f7,43983,1733811247584, state=OPENING 2024-12-10T06:14:10,904 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T06:14:10,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:10,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:14:10,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:14:10,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:14:10,909 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T06:14:10,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:14:11,092 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T06:14:11,096 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50159, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T06:14:11,107 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T06:14:11,107 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:14:11,107 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T06:14:11,111 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c42c682b34f7%2C43983%2C1733811247584.meta, suffix=.meta, logDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/WALs/c42c682b34f7,43983,1733811247584, archiveDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/oldWALs, maxLogs=32 2024-12-10T06:14:11,129 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/WALs/c42c682b34f7,43983,1733811247584/c42c682b34f7%2C43983%2C1733811247584.meta.1733811251113.meta, exclude list is [], retry=0 2024-12-10T06:14:11,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35903,DS-5636f389-6925-4170-b0d5-678af6414ef2,DISK] 2024-12-10T06:14:11,136 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/WALs/c42c682b34f7,43983,1733811247584/c42c682b34f7%2C43983%2C1733811247584.meta.1733811251113.meta 2024-12-10T06:14:11,137 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45429:45429)] 2024-12-10T06:14:11,137 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:14:11,139 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T06:14:11,142 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T06:14:11,147 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T06:14:11,152 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T06:14:11,153 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:14:11,153 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T06:14:11,153 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T06:14:11,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T06:14:11,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T06:14:11,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:11,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:14:11,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T06:14:11,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T06:14:11,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:11,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:14:11,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T06:14:11,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T06:14:11,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:11,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:14:11,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T06:14:11,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T06:14:11,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:11,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:14:11,166 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T06:14:11,167 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740 2024-12-10T06:14:11,170 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740 2024-12-10T06:14:11,172 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T06:14:11,172 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T06:14:11,173 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:14:11,176 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T06:14:11,178 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71913065, jitterRate=0.0715881735086441}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:14:11,178 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T06:14:11,179 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733811251153Writing region info on filesystem at 1733811251154 (+1 ms)Initializing all the Stores at 1733811251156 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733811251156Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733811251156Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811251156Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733811251156Cleaning up temporary data from old regions at 1733811251172 (+16 ms)Running coprocessor post-open hooks at 1733811251178 (+6 ms)Region opened successfully at 1733811251179 (+1 ms) 2024-12-10T06:14:11,187 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733811251083 2024-12-10T06:14:11,200 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T06:14:11,200 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T06:14:11,202 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:11,204 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c42c682b34f7,43983,1733811247584, state=OPEN 2024-12-10T06:14:11,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T06:14:11,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T06:14:11,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:14:11,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:14:11,210 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:11,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T06:14:11,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c42c682b34f7,43983,1733811247584 in 299 msec 2024-12-10T06:14:11,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T06:14:11,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 765 msec 2024-12-10T06:14:11,223 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T06:14:11,223 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T06:14:11,245 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:11,246 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:11,268 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:11,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34807, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:11,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0760 sec 2024-12-10T06:14:11,291 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733811251291, completionTime=-1 2024-12-10T06:14:11,294 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-10T06:14:11,294 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T06:14:11,344 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-10T06:14:11,344 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733811311344 2024-12-10T06:14:11,344 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733811371344 2024-12-10T06:14:11,345 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 50 msec 2024-12-10T06:14:11,347 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,37415,1733811246834-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:11,347 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,37415,1733811246834-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:11,347 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,37415,1733811246834-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:11,349 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c42c682b34f7:37415, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:11,349 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:11,358 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:11,363 DEBUG [master/c42c682b34f7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T06:14:11,385 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.655sec 2024-12-10T06:14:11,387 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T06:14:11,388 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T06:14:11,389 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T06:14:11,390 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T06:14:11,390 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T06:14:11,391 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,37415,1733811246834-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T06:14:11,391 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,37415,1733811246834-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T06:14:11,399 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T06:14:11,400 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T06:14:11,400 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,37415,1733811246834-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:14:11,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:11,495 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T06:14:11,496 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T06:14:11,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:11,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:11,508 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:11,519 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:11,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:11,522 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:11,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:11,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c42c682b34f7,37415,1733811246834 2024-12-10T06:14:11,563 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=287, ProcessCount=11, AvailableMemoryMB=7015 2024-12-10T06:14:11,571 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T06:14:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c42c682b34f7,37415,1733811246834 2024-12-10T06:14:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@404c298d 2024-12-10T06:14:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T06:14:11,582 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T06:14:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T06:14:11,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:14:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:11,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T06:14:11,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-10T06:14:11,600 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:11,602 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T06:14:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:14:11,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741835_1011 (size=1061) 2024-12-10T06:14:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:14:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:14:12,017 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6 2024-12-10T06:14:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741836_1012 (size=53) 2024-12-10T06:14:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:14:12,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:14:12,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 91cf7a8c24ad1cfeb344be5ce72aa8a5, disabling compactions & flushes 2024-12-10T06:14:12,428 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:12,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:12,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. after waiting 0 ms 2024-12-10T06:14:12,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:12,429 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:12,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: Waiting for close lock at 1733811252428Disabling compacts and flushes for region at 1733811252428Disabling writes for close at 1733811252429 (+1 ms)Writing region close event to WAL at 1733811252429Closed at 1733811252429 2024-12-10T06:14:12,432 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T06:14:12,437 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733811252432"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733811252432"}]},"ts":"1733811252432"} 2024-12-10T06:14:12,442 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T06:14:12,444 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T06:14:12,446 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811252444"}]},"ts":"1733811252444"} 2024-12-10T06:14:12,450 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T06:14:12,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91cf7a8c24ad1cfeb344be5ce72aa8a5, ASSIGN}] 2024-12-10T06:14:12,454 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91cf7a8c24ad1cfeb344be5ce72aa8a5, ASSIGN 2024-12-10T06:14:12,456 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91cf7a8c24ad1cfeb344be5ce72aa8a5, ASSIGN; state=OFFLINE, location=c42c682b34f7,43983,1733811247584; forceNewPlan=false, retain=false 2024-12-10T06:14:12,607 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=91cf7a8c24ad1cfeb344be5ce72aa8a5, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:12,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91cf7a8c24ad1cfeb344be5ce72aa8a5, ASSIGN because future has completed 2024-12-10T06:14:12,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:14:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:14:12,775 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:12,775 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:14:12,776 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,776 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:14:12,777 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,777 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,779 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,783 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:14:12,784 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91cf7a8c24ad1cfeb344be5ce72aa8a5 columnFamilyName A 2024-12-10T06:14:12,784 DEBUG [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:12,785 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] regionserver.HStore(327): Store=91cf7a8c24ad1cfeb344be5ce72aa8a5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:12,786 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,788 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:14:12,789 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91cf7a8c24ad1cfeb344be5ce72aa8a5 columnFamilyName B 2024-12-10T06:14:12,789 DEBUG [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:12,790 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] regionserver.HStore(327): Store=91cf7a8c24ad1cfeb344be5ce72aa8a5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:12,790 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,792 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:14:12,793 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91cf7a8c24ad1cfeb344be5ce72aa8a5 columnFamilyName C 2024-12-10T06:14:12,793 DEBUG [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:12,794 INFO [StoreOpener-91cf7a8c24ad1cfeb344be5ce72aa8a5-1 {}] regionserver.HStore(327): Store=91cf7a8c24ad1cfeb344be5ce72aa8a5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:12,795 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:12,795 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,797 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,798 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,800 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,800 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,801 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:14:12,804 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,809 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:14:12,809 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 91cf7a8c24ad1cfeb344be5ce72aa8a5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68736975, jitterRate=0.0242607444524765}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:14:12,810 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:12,811 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: Running coprocessor pre-open hook at 1733811252777Writing region info on filesystem at 1733811252777Initializing all the Stores at 1733811252779 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811252779Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811252779Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811252779Cleaning up temporary data from old regions at 1733811252800 (+21 ms)Running coprocessor post-open hooks at 1733811252810 (+10 ms)Region opened successfully at 1733811252810 2024-12-10T06:14:12,813 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., pid=6, masterSystemTime=1733811252767 2024-12-10T06:14:12,817 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:12,817 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:12,819 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=91cf7a8c24ad1cfeb344be5ce72aa8a5, regionState=OPEN, openSeqNum=2, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:12,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:14:12,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T06:14:12,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 in 213 msec 2024-12-10T06:14:12,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T06:14:12,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91cf7a8c24ad1cfeb344be5ce72aa8a5, ASSIGN in 378 msec 2024-12-10T06:14:12,835 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T06:14:12,836 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811252836"}]},"ts":"1733811252836"} 2024-12-10T06:14:12,839 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T06:14:12,841 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T06:14:12,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2510 sec 2024-12-10T06:14:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:14:13,757 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:13,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,765 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f6a966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc7db18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,771 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,781 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T06:14:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c42c682b34f7,37415,1733811246834 2024-12-10T06:14:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f70fa58 2024-12-10T06:14:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T06:14:13,788 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T06:14:13,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,794 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39deefb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,808 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ccbe83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,812 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,818 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b0e30d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,826 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4545009e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,830 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31097510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,838 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,841 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47cbee8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,847 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b036a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,850 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129161b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89bfc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d3547a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb5cb2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,864 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:13,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:13,868 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b2908d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:13,870 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cfd6776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:13,878 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,878 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,879 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,879 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:14:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:14:13,894 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,895 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:14:13,898 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,898 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:14:13,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:14:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:14:13,911 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,917 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,919 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,919 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:14:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-10T06:14:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:13,929 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T06:14:13,931 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:13,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:13,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:14:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:14:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:14,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T06:14:14,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:14,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:14,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:14,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:14,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:14,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T06:14:14,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:14,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:14,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811314109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811314114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811314116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811314120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811314139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/713ea3f7a5144e31923a21a444b317e9 is 50, key is test_row_0/A:col10/1733811253998/Put/seqid=0 2024-12-10T06:14:14,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741837_1013 (size=16681) 2024-12-10T06:14:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/713ea3f7a5144e31923a21a444b317e9 2024-12-10T06:14:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T06:14:14,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811314276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811314277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8c045b698b1241ed9c21c5208208281c is 50, key is test_row_0/B:col10/1733811253998/Put/seqid=0 2024-12-10T06:14:14,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811314287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:14,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811314288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811314288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741838_1014 (size=12001) 2024-12-10T06:14:14,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8c045b698b1241ed9c21c5208208281c 2024-12-10T06:14:14,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0ed9ec52dc8448eaa36ed0e965df1fc0 is 50, key is test_row_0/C:col10/1733811253998/Put/seqid=0 2024-12-10T06:14:14,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741839_1015 (size=12001) 2024-12-10T06:14:14,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0ed9ec52dc8448eaa36ed0e965df1fc0 2024-12-10T06:14:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/713ea3f7a5144e31923a21a444b317e9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/713ea3f7a5144e31923a21a444b317e9 2024-12-10T06:14:14,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:14,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/713ea3f7a5144e31923a21a444b317e9, entries=250, sequenceid=14, filesize=16.3 K 2024-12-10T06:14:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8c045b698b1241ed9c21c5208208281c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8c045b698b1241ed9c21c5208208281c 2024-12-10T06:14:14,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8c045b698b1241ed9c21c5208208281c, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T06:14:14,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0ed9ec52dc8448eaa36ed0e965df1fc0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0ed9ec52dc8448eaa36ed0e965df1fc0 2024-12-10T06:14:14,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0ed9ec52dc8448eaa36ed0e965df1fc0, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T06:14:14,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 499ms, sequenceid=14, compaction requested=false 2024-12-10T06:14:14,517 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T06:14:14,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:14,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T06:14:14,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:14,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811254106, max timestamp=1733811254513Num uniques 859; 2024-12-10T06:14:14,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:14,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:14,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:14:14,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:14,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:14,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:14,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/577e72e833f143f9b5744ecbc8e31640 is 50, key is test_row_0/A:col10/1733811254513/Put/seqid=0 2024-12-10T06:14:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T06:14:14,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741840_1016 (size=12001) 2024-12-10T06:14:14,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:14,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:14,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811314591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811314591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811314596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811314635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811314635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811314746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811314747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811314747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811314767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811314767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:14,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:14,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:14,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:14,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:14,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/577e72e833f143f9b5744ecbc8e31640 2024-12-10T06:14:14,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811314966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811314966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811314968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/d651a49ee8ab411a9b6df157507e7b74 is 50, key is test_row_0/B:col10/1733811254513/Put/seqid=0 2024-12-10T06:14:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811314987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811314987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741841_1017 (size=12001) 2024-12-10T06:14:15,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/d651a49ee8ab411a9b6df157507e7b74 2024-12-10T06:14:15,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/dd693241781d46fdaa1bb40b4067d59e is 50, key is test_row_0/C:col10/1733811254513/Put/seqid=0 2024-12-10T06:14:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T06:14:15,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741842_1018 (size=12001) 2024-12-10T06:14:15,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:15,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:15,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:15,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:15,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:15,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811315289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811315296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811315296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811315317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811315317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:15,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:15,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:15,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:15,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/dd693241781d46fdaa1bb40b4067d59e 2024-12-10T06:14:15,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/577e72e833f143f9b5744ecbc8e31640 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/577e72e833f143f9b5744ecbc8e31640 2024-12-10T06:14:15,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/577e72e833f143f9b5744ecbc8e31640, entries=150, sequenceid=35, filesize=11.7 K 2024-12-10T06:14:15,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/d651a49ee8ab411a9b6df157507e7b74 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/d651a49ee8ab411a9b6df157507e7b74 2024-12-10T06:14:15,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/d651a49ee8ab411a9b6df157507e7b74, entries=150, sequenceid=35, filesize=11.7 K 2024-12-10T06:14:15,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/dd693241781d46fdaa1bb40b4067d59e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/dd693241781d46fdaa1bb40b4067d59e 2024-12-10T06:14:15,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:15,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/dd693241781d46fdaa1bb40b4067d59e, entries=150, sequenceid=35, filesize=11.7 K 2024-12-10T06:14:15,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:15,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:15,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:15,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:15,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.69 KB/93890, heapSize ~267.14 KB/273552, currentSize=87.22 KB/89310 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 1042ms, sequenceid=35, compaction requested=false 2024-12-10T06:14:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:15,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T06:14:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:15,724 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:14:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/2a924a6225db4f3baba79f9275723211 is 50, key is test_row_0/A:col10/1733811254599/Put/seqid=0 2024-12-10T06:14:15,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741843_1019 (size=12001) 2024-12-10T06:14:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:15,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811315852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811315852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811315862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811315866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811315866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811315976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811315977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811315977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811315986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:15,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811315986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:15,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T06:14:16,157 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/2a924a6225db4f3baba79f9275723211 2024-12-10T06:14:16,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3418c204d92b411597a890b867b3f9a7 is 50, key is test_row_0/B:col10/1733811254599/Put/seqid=0 2024-12-10T06:14:16,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811316196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811316197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811316198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811316199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811316199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741844_1020 (size=12001) 2024-12-10T06:14:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811316505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811316506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811316506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811316507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811316507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:16,609 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3418c204d92b411597a890b867b3f9a7 2024-12-10T06:14:16,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/8b7cbbd7d942442585103ba5a2bd9f29 is 50, key is test_row_0/C:col10/1733811254599/Put/seqid=0 2024-12-10T06:14:16,667 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:14:16,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741845_1021 (size=12001) 2024-12-10T06:14:16,676 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/8b7cbbd7d942442585103ba5a2bd9f29 2024-12-10T06:14:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/2a924a6225db4f3baba79f9275723211 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/2a924a6225db4f3baba79f9275723211 2024-12-10T06:14:16,707 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/2a924a6225db4f3baba79f9275723211, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T06:14:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3418c204d92b411597a890b867b3f9a7 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3418c204d92b411597a890b867b3f9a7 2024-12-10T06:14:16,722 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3418c204d92b411597a890b867b3f9a7, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T06:14:16,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/8b7cbbd7d942442585103ba5a2bd9f29 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8b7cbbd7d942442585103ba5a2bd9f29 2024-12-10T06:14:16,737 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T06:14:16,743 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8b7cbbd7d942442585103ba5a2bd9f29, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T06:14:16,746 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 1022ms, sequenceid=51, compaction requested=true 2024-12-10T06:14:16,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:16,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:16,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-10T06:14:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-10T06:14:16,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T06:14:16,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8200 sec 2024-12-10T06:14:16,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.8380 sec 2024-12-10T06:14:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:17,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T06:14:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:17,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/4aa4a313063a4c33a3241060121ff3f2 is 50, key is test_row_0/A:col10/1733811257034/Put/seqid=0 2024-12-10T06:14:17,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741846_1022 (size=12001) 2024-12-10T06:14:17,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811317056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811317058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811317059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811317062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/4aa4a313063a4c33a3241060121ff3f2 2024-12-10T06:14:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811317067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/c8d68aefc43b4e7bac8efd41fa0dcc9d is 50, key is test_row_0/B:col10/1733811257034/Put/seqid=0 2024-12-10T06:14:17,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741847_1023 (size=12001) 2024-12-10T06:14:17,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811317177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811317177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811317177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811317177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811317186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,281 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T06:14:17,281 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T06:14:17,289 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T06:14:17,289 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T06:14:17,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T06:14:17,294 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T06:14:17,303 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T06:14:17,303 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T06:14:17,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811317385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811317395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811317396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811317396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811317396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/c8d68aefc43b4e7bac8efd41fa0dcc9d 2024-12-10T06:14:17,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/bc54847c7ccd4df786645b734df35e09 is 50, key is test_row_0/C:col10/1733811257034/Put/seqid=0 2024-12-10T06:14:17,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741848_1024 (size=12001) 2024-12-10T06:14:17,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811317695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811317716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811317716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811317716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811317717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:17,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/bc54847c7ccd4df786645b734df35e09 2024-12-10T06:14:17,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/4aa4a313063a4c33a3241060121ff3f2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4aa4a313063a4c33a3241060121ff3f2 2024-12-10T06:14:17,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4aa4a313063a4c33a3241060121ff3f2, entries=150, sequenceid=71, filesize=11.7 K 2024-12-10T06:14:17,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/c8d68aefc43b4e7bac8efd41fa0dcc9d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/c8d68aefc43b4e7bac8efd41fa0dcc9d 2024-12-10T06:14:18,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/c8d68aefc43b4e7bac8efd41fa0dcc9d, entries=150, sequenceid=71, filesize=11.7 K 2024-12-10T06:14:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/bc54847c7ccd4df786645b734df35e09 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/bc54847c7ccd4df786645b734df35e09 2024-12-10T06:14:18,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/bc54847c7ccd4df786645b734df35e09, entries=150, sequenceid=71, filesize=11.7 K 2024-12-10T06:14:18,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 984ms, sequenceid=71, compaction requested=true 2024-12-10T06:14:18,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:18,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:18,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:18,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:18,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:18,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:18,037 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:18,038 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:18,045 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:18,047 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/B is initiating minor compaction (all files) 2024-12-10T06:14:18,048 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/B in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,048 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8c045b698b1241ed9c21c5208208281c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/d651a49ee8ab411a9b6df157507e7b74, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3418c204d92b411597a890b867b3f9a7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/c8d68aefc43b4e7bac8efd41fa0dcc9d] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=46.9 K 2024-12-10T06:14:18,050 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c045b698b1241ed9c21c5208208281c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733811253971 2024-12-10T06:14:18,051 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d651a49ee8ab411a9b6df157507e7b74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733811254106 2024-12-10T06:14:18,052 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 3418c204d92b411597a890b867b3f9a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811254554 2024-12-10T06:14:18,053 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting c8d68aefc43b4e7bac8efd41fa0dcc9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733811257031 2024-12-10T06:14:18,056 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:18,057 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/A is initiating minor compaction (all files) 2024-12-10T06:14:18,057 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/A in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,066 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/713ea3f7a5144e31923a21a444b317e9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/577e72e833f143f9b5744ecbc8e31640, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/2a924a6225db4f3baba79f9275723211, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4aa4a313063a4c33a3241060121ff3f2] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=51.4 K 2024-12-10T06:14:18,066 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 713ea3f7a5144e31923a21a444b317e9, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733811253971 2024-12-10T06:14:18,067 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 577e72e833f143f9b5744ecbc8e31640, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733811254106 2024-12-10T06:14:18,069 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a924a6225db4f3baba79f9275723211, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811254554 2024-12-10T06:14:18,070 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aa4a313063a4c33a3241060121ff3f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733811257031 2024-12-10T06:14:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T06:14:18,096 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:18,096 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-10T06:14:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T06:14:18,100 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:18,102 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:18,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:18,102 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#B#compaction#12 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:18,103 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8046ae8979ff479eb5d1fb38d3b6dae0 is 50, key is test_row_0/B:col10/1733811257034/Put/seqid=0 2024-12-10T06:14:18,113 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#A#compaction#13 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:18,114 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/311a874dcfc546029655781058751ab8 is 50, key is test_row_0/A:col10/1733811257034/Put/seqid=0 2024-12-10T06:14:18,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741849_1025 (size=12139) 2024-12-10T06:14:18,150 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8046ae8979ff479eb5d1fb38d3b6dae0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8046ae8979ff479eb5d1fb38d3b6dae0 2024-12-10T06:14:18,179 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/B of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 8046ae8979ff479eb5d1fb38d3b6dae0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:18,179 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:18,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741850_1026 (size=12139) 2024-12-10T06:14:18,182 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/B, priority=12, startTime=1733811258036; duration=0sec 2024-12-10T06:14:18,183 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:18,183 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:B 2024-12-10T06:14:18,183 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:18,186 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:18,186 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/C is initiating minor compaction (all files) 2024-12-10T06:14:18,187 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/C in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,187 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0ed9ec52dc8448eaa36ed0e965df1fc0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/dd693241781d46fdaa1bb40b4067d59e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8b7cbbd7d942442585103ba5a2bd9f29, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/bc54847c7ccd4df786645b734df35e09] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=46.9 K 2024-12-10T06:14:18,188 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ed9ec52dc8448eaa36ed0e965df1fc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733811253971 2024-12-10T06:14:18,192 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting dd693241781d46fdaa1bb40b4067d59e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733811254106 2024-12-10T06:14:18,195 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b7cbbd7d942442585103ba5a2bd9f29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811254554 2024-12-10T06:14:18,195 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting bc54847c7ccd4df786645b734df35e09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733811257031 2024-12-10T06:14:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T06:14:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:18,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T06:14:18,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:18,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:18,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:18,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:18,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:18,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:18,230 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#C#compaction#14 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:18,231 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/df47a1313b624d44bfe24b81317252d4 is 50, key is test_row_0/C:col10/1733811257034/Put/seqid=0 2024-12-10T06:14:18,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/49a8fd7e9bfb40508356757cd0420396 is 50, key is test_row_0/A:col10/1733811257055/Put/seqid=0 2024-12-10T06:14:18,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T06:14:18,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:18,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741851_1027 (size=12139) 2024-12-10T06:14:18,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741852_1028 (size=12001) 2024-12-10T06:14:18,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/49a8fd7e9bfb40508356757cd0420396 2024-12-10T06:14:18,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811318270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811318287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811258262, max timestamp=1733811258289Num uniques 859; 2024-12-10T06:14:18,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:14:18,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/66545074372f4754839dc673cf33c911 is 50, key is test_row_0/B:col10/1733811257055/Put/seqid=0 2024-12-10T06:14:18,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811258262, max timestamp=1733811258289Num uniques 859; 2024-12-10T06:14:18,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:14:18,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811258262, max timestamp=1733811258289Num uniques 859; 2024-12-10T06:14:18,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:14:18,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741853_1029 (size=12001) 2024-12-10T06:14:18,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811318362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811318366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811318369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811318395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811318406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T06:14:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:18,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T06:14:18,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811318486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811318487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811318487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T06:14:18,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:18,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,593 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/311a874dcfc546029655781058751ab8 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/311a874dcfc546029655781058751ab8 2024-12-10T06:14:18,611 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/A of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 311a874dcfc546029655781058751ab8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:18,611 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:18,611 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/A, priority=12, startTime=1733811258023; duration=0sec 2024-12-10T06:14:18,611 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:18,611 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:A 2024-12-10T06:14:18,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811318616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811318647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,677 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/df47a1313b624d44bfe24b81317252d4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/df47a1313b624d44bfe24b81317252d4 2024-12-10T06:14:18,688 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/C of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into df47a1313b624d44bfe24b81317252d4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:18,688 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:18,688 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/C, priority=12, startTime=1733811258036; duration=0sec 2024-12-10T06:14:18,688 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:18,688 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:C 2024-12-10T06:14:18,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811318705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811318705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811318705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T06:14:18,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T06:14:18,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:18,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:18,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/66545074372f4754839dc673cf33c911 2024-12-10T06:14:18,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/9ff7109bb51c4056a39479b860b14036 is 50, key is test_row_0/C:col10/1733811257055/Put/seqid=0 2024-12-10T06:14:18,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741854_1030 (size=12001) 2024-12-10T06:14:18,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/9ff7109bb51c4056a39479b860b14036 2024-12-10T06:14:18,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/49a8fd7e9bfb40508356757cd0420396 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/49a8fd7e9bfb40508356757cd0420396 2024-12-10T06:14:18,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/49a8fd7e9bfb40508356757cd0420396, entries=150, sequenceid=85, filesize=11.7 K 2024-12-10T06:14:18,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/66545074372f4754839dc673cf33c911 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/66545074372f4754839dc673cf33c911 2024-12-10T06:14:18,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/66545074372f4754839dc673cf33c911, entries=150, sequenceid=85, filesize=11.7 K 2024-12-10T06:14:18,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/9ff7109bb51c4056a39479b860b14036 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/9ff7109bb51c4056a39479b860b14036 2024-12-10T06:14:18,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/9ff7109bb51c4056a39479b860b14036, entries=150, sequenceid=85, filesize=11.7 K 2024-12-10T06:14:18,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 600ms, sequenceid=85, compaction requested=false 2024-12-10T06:14:18,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:18,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T06:14:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:18,882 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=114.05 KB heapSize=325.22 KB 2024-12-10T06:14:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/dc79dd94682746a9bfaecf606ed19448 is 50, key is test_row_0/A:col10/1733811258366/Put/seqid=0 2024-12-10T06:14:18,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741855_1031 (size=12001) 2024-12-10T06:14:18,905 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/dc79dd94682746a9bfaecf606ed19448 2024-12-10T06:14:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3db7f71733814f2297adbb37fce68e02 is 50, key is test_row_0/B:col10/1733811258366/Put/seqid=0 2024-12-10T06:14:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:18,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:18,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741856_1032 (size=12001) 2024-12-10T06:14:18,968 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3db7f71733814f2297adbb37fce68e02 2024-12-10T06:14:19,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/2bc5ac6f4b6e4d518143f20b29dd006a is 50, key is test_row_0/C:col10/1733811258366/Put/seqid=0 2024-12-10T06:14:19,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811319016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811319017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811319017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741857_1033 (size=12001) 2024-12-10T06:14:19,030 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/2bc5ac6f4b6e4d518143f20b29dd006a 2024-12-10T06:14:19,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811319021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811319023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/dc79dd94682746a9bfaecf606ed19448 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dc79dd94682746a9bfaecf606ed19448 2024-12-10T06:14:19,055 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dc79dd94682746a9bfaecf606ed19448, entries=150, sequenceid=121, filesize=11.7 K 2024-12-10T06:14:19,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3db7f71733814f2297adbb37fce68e02 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3db7f71733814f2297adbb37fce68e02 2024-12-10T06:14:19,096 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3db7f71733814f2297adbb37fce68e02, entries=150, sequenceid=121, filesize=11.7 K 2024-12-10T06:14:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/2bc5ac6f4b6e4d518143f20b29dd006a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2bc5ac6f4b6e4d518143f20b29dd006a 2024-12-10T06:14:19,110 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2bc5ac6f4b6e4d518143f20b29dd006a, entries=150, sequenceid=121, filesize=11.7 K 2024-12-10T06:14:19,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~325.17 KB/332976, currentSize=67.09 KB/68700 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 232ms, sequenceid=121, compaction requested=true 2024-12-10T06:14:19,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:19,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-10T06:14:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-10T06:14:19,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-10T06:14:19,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-12-10T06:14:19,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0260 sec 2024-12-10T06:14:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:19,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T06:14:19,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:19,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:19,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:19,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:19,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:19,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:19,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/d8f8cc135aec4b0e9083d685d5c59b67 is 50, key is test_row_0/A:col10/1733811259020/Put/seqid=0 2024-12-10T06:14:19,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741858_1034 (size=12151) 2024-12-10T06:14:19,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/d8f8cc135aec4b0e9083d685d5c59b67 2024-12-10T06:14:19,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/2de0a10630514d3fbaa147beecd1ecda is 50, key is test_row_0/B:col10/1733811259020/Put/seqid=0 2024-12-10T06:14:19,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811319193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811319201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811319205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T06:14:19,236 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:19,238 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:19,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741859_1035 (size=12151) 2024-12-10T06:14:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-10T06:14:19,242 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:19,246 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:19,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T06:14:19,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811319316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811319316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811319317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T06:14:19,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:19,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:19,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811319526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811319526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811319528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811319536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811319536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T06:14:19,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/2de0a10630514d3fbaa147beecd1ecda 2024-12-10T06:14:19,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/79a50f782fa74c85a26a8555fbee5669 is 50, key is test_row_0/C:col10/1733811259020/Put/seqid=0 2024-12-10T06:14:19,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741860_1036 (size=12151) 2024-12-10T06:14:19,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/79a50f782fa74c85a26a8555fbee5669 2024-12-10T06:14:19,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/d8f8cc135aec4b0e9083d685d5c59b67 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d8f8cc135aec4b0e9083d685d5c59b67 2024-12-10T06:14:19,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d8f8cc135aec4b0e9083d685d5c59b67, entries=150, sequenceid=136, filesize=11.9 K 2024-12-10T06:14:19,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/2de0a10630514d3fbaa147beecd1ecda as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/2de0a10630514d3fbaa147beecd1ecda 2024-12-10T06:14:19,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:19,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:19,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/2de0a10630514d3fbaa147beecd1ecda, entries=150, sequenceid=136, filesize=11.9 K 2024-12-10T06:14:19,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/79a50f782fa74c85a26a8555fbee5669 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/79a50f782fa74c85a26a8555fbee5669 2024-12-10T06:14:19,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/79a50f782fa74c85a26a8555fbee5669, entries=150, sequenceid=136, filesize=11.9 K 2024-12-10T06:14:19,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 595ms, sequenceid=136, compaction requested=true 2024-12-10T06:14:19,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:19,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:19,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:19,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:19,735 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:19,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:19,735 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:19,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:19,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:19,737 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:19,737 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/B is initiating minor compaction (all files) 2024-12-10T06:14:19,737 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:19,737 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/B in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,737 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/A is initiating minor compaction (all files) 2024-12-10T06:14:19,738 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/A in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,738 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8046ae8979ff479eb5d1fb38d3b6dae0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/66545074372f4754839dc673cf33c911, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3db7f71733814f2297adbb37fce68e02, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/2de0a10630514d3fbaa147beecd1ecda] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=47.2 K 2024-12-10T06:14:19,738 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/311a874dcfc546029655781058751ab8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/49a8fd7e9bfb40508356757cd0420396, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dc79dd94682746a9bfaecf606ed19448, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d8f8cc135aec4b0e9083d685d5c59b67] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=47.2 K 2024-12-10T06:14:19,740 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8046ae8979ff479eb5d1fb38d3b6dae0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733811257031 2024-12-10T06:14:19,740 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 311a874dcfc546029655781058751ab8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733811257031 2024-12-10T06:14:19,741 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49a8fd7e9bfb40508356757cd0420396, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733811257055 2024-12-10T06:14:19,741 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 66545074372f4754839dc673cf33c911, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733811257055 2024-12-10T06:14:19,743 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc79dd94682746a9bfaecf606ed19448, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733811258346 2024-12-10T06:14:19,743 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db7f71733814f2297adbb37fce68e02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733811258346 2024-12-10T06:14:19,743 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2de0a10630514d3fbaa147beecd1ecda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733811259020 2024-12-10T06:14:19,743 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f8cc135aec4b0e9083d685d5c59b67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733811259020 2024-12-10T06:14:19,776 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#B#compaction#24 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:19,777 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/6f2a36cbdeb34ba688b4dfa0d82060af is 50, key is test_row_0/B:col10/1733811259020/Put/seqid=0 2024-12-10T06:14:19,781 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#A#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:19,782 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/121bf73ad88347d29bcd48e95e127a45 is 50, key is test_row_0/A:col10/1733811259020/Put/seqid=0 2024-12-10T06:14:19,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741861_1037 (size=12425) 2024-12-10T06:14:19,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741862_1038 (size=12425) 2024-12-10T06:14:19,823 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/6f2a36cbdeb34ba688b4dfa0d82060af as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/6f2a36cbdeb34ba688b4dfa0d82060af 2024-12-10T06:14:19,833 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/B of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 6f2a36cbdeb34ba688b4dfa0d82060af(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:19,833 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:19,833 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/B, priority=12, startTime=1733811259735; duration=0sec 2024-12-10T06:14:19,834 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:19,834 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:B 2024-12-10T06:14:19,834 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:19,836 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:19,836 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/C is initiating minor compaction (all files) 2024-12-10T06:14:19,836 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/C in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,837 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/df47a1313b624d44bfe24b81317252d4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/9ff7109bb51c4056a39479b860b14036, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2bc5ac6f4b6e4d518143f20b29dd006a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/79a50f782fa74c85a26a8555fbee5669] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=47.2 K 2024-12-10T06:14:19,837 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting df47a1313b624d44bfe24b81317252d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733811257031 2024-12-10T06:14:19,838 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ff7109bb51c4056a39479b860b14036, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733811257055 2024-12-10T06:14:19,840 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc5ac6f4b6e4d518143f20b29dd006a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733811258346 2024-12-10T06:14:19,844 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a50f782fa74c85a26a8555fbee5669, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733811259020 2024-12-10T06:14:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:19,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T06:14:19,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:19,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:19,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:19,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:19,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:19,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:19,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/e757c74ffad54262b80a6d4012b8d9f9 is 50, key is test_row_0/A:col10/1733811259199/Put/seqid=0 2024-12-10T06:14:19,868 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#C#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:19,869 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/8bf7063a0d1e4ac8ad3bf5b93244c00f is 50, key is test_row_0/C:col10/1733811259020/Put/seqid=0 2024-12-10T06:14:19,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:19,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:19,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:19,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T06:14:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811319873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811319874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811319878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741864_1040 (size=12425) 2024-12-10T06:14:19,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741863_1039 (size=14541) 2024-12-10T06:14:19,902 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/8bf7063a0d1e4ac8ad3bf5b93244c00f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8bf7063a0d1e4ac8ad3bf5b93244c00f 2024-12-10T06:14:19,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/e757c74ffad54262b80a6d4012b8d9f9 2024-12-10T06:14:19,916 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/C of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 8bf7063a0d1e4ac8ad3bf5b93244c00f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:19,917 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:19,917 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/C, priority=12, startTime=1733811259735; duration=0sec 2024-12-10T06:14:19,918 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:19,918 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:C 2024-12-10T06:14:19,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/cc93bb07204b4be48be5cb6df204d33e is 50, key is test_row_0/B:col10/1733811259199/Put/seqid=0 2024-12-10T06:14:19,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741865_1041 (size=12151) 2024-12-10T06:14:19,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811319986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811319986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:19,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811319995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:20,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:20,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:20,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:20,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811320205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:20,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811320206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:20,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811320206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:20,227 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/121bf73ad88347d29bcd48e95e127a45 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/121bf73ad88347d29bcd48e95e127a45 2024-12-10T06:14:20,238 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/A of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 121bf73ad88347d29bcd48e95e127a45(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:20,238 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:20,238 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/A, priority=12, startTime=1733811259735; duration=0sec 2024-12-10T06:14:20,238 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:20,238 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:A 2024-12-10T06:14:20,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/cc93bb07204b4be48be5cb6df204d33e 2024-12-10T06:14:20,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0d850dc92d6f45a8877f0ba17477da0a is 50, key is test_row_0/C:col10/1733811259199/Put/seqid=0 2024-12-10T06:14:20,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741866_1042 (size=12151) 2024-12-10T06:14:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T06:14:20,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:20,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811320516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811320517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811320517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:20,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811320556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:20,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811320556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:20,563 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 91cf7a8c24ad1cfeb344be5ce72aa8a5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:20,565 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 91cf7a8c24ad1cfeb344be5ce72aa8a5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:20,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0d850dc92d6f45a8877f0ba17477da0a 2024-12-10T06:14:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/e757c74ffad54262b80a6d4012b8d9f9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/e757c74ffad54262b80a6d4012b8d9f9 2024-12-10T06:14:20,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/e757c74ffad54262b80a6d4012b8d9f9, entries=200, sequenceid=155, filesize=14.2 K 2024-12-10T06:14:20,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/cc93bb07204b4be48be5cb6df204d33e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/cc93bb07204b4be48be5cb6df204d33e 2024-12-10T06:14:20,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:20,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/cc93bb07204b4be48be5cb6df204d33e, entries=150, sequenceid=155, filesize=11.9 K 2024-12-10T06:14:20,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0d850dc92d6f45a8877f0ba17477da0a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0d850dc92d6f45a8877f0ba17477da0a 2024-12-10T06:14:20,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0d850dc92d6f45a8877f0ba17477da0a, entries=150, sequenceid=155, filesize=11.9 K 2024-12-10T06:14:20,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 997ms, sequenceid=155, compaction requested=false 2024-12-10T06:14:20,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:20,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T06:14:20,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:20,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T06:14:20,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:20,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:20,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/dfcbcde06bd841ca93d456053803a2b6 is 50, key is test_row_0/A:col10/1733811259865/Put/seqid=0 2024-12-10T06:14:21,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741867_1043 (size=12151) 2024-12-10T06:14:21,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/dfcbcde06bd841ca93d456053803a2b6 2024-12-10T06:14:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:21,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/35b59f611e324e8ea8e8315d5c130dd5 is 50, key is test_row_0/B:col10/1733811259865/Put/seqid=0 2024-12-10T06:14:21,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741868_1044 (size=12151) 2024-12-10T06:14:21,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811321101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811321103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811321107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811321215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811321215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811321215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T06:14:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811321426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811321426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811321426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,457 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/35b59f611e324e8ea8e8315d5c130dd5 2024-12-10T06:14:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/7ecfa963fbb1438e83ec2121c3f54789 is 50, key is test_row_0/C:col10/1733811259865/Put/seqid=0 2024-12-10T06:14:21,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741869_1045 (size=12151) 2024-12-10T06:14:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811321736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811321736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811321736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:21,894 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/7ecfa963fbb1438e83ec2121c3f54789 2024-12-10T06:14:21,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/dfcbcde06bd841ca93d456053803a2b6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dfcbcde06bd841ca93d456053803a2b6 2024-12-10T06:14:21,911 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dfcbcde06bd841ca93d456053803a2b6, entries=150, sequenceid=171, filesize=11.9 K 2024-12-10T06:14:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/35b59f611e324e8ea8e8315d5c130dd5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/35b59f611e324e8ea8e8315d5c130dd5 2024-12-10T06:14:21,919 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/35b59f611e324e8ea8e8315d5c130dd5, entries=150, sequenceid=171, filesize=11.9 K 2024-12-10T06:14:21,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/7ecfa963fbb1438e83ec2121c3f54789 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/7ecfa963fbb1438e83ec2121c3f54789 2024-12-10T06:14:21,927 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/7ecfa963fbb1438e83ec2121c3f54789, entries=150, sequenceid=171, filesize=11.9 K 2024-12-10T06:14:21,928 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 963ms, sequenceid=171, compaction requested=true 2024-12-10T06:14:21,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:21,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:21,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-10T06:14:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-10T06:14:21,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-10T06:14:21,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6840 sec 2024-12-10T06:14:21,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.6990 sec 2024-12-10T06:14:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:22,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T06:14:22,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:22,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/d52eec660ce24f7e9d0d486c3a1883f6 is 50, key is test_row_0/A:col10/1733811261101/Put/seqid=0 2024-12-10T06:14:22,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741870_1046 (size=14541) 2024-12-10T06:14:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811322278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/d52eec660ce24f7e9d0d486c3a1883f6 2024-12-10T06:14:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811322279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811322281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/740dcc45f6c944f7b3cf11535962e82a is 50, key is test_row_0/B:col10/1733811261101/Put/seqid=0 2024-12-10T06:14:22,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741871_1047 (size=12151) 2024-12-10T06:14:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811322396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811322405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811322405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811322576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,578 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 91cf7a8c24ad1cfeb344be5ce72aa8a5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811322585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,589 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 91cf7a8c24ad1cfeb344be5ce72aa8a5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811322605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811322616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811322616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/740dcc45f6c944f7b3cf11535962e82a 2024-12-10T06:14:22,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/03f7b589ccec49628f15849f74702409 is 50, key is test_row_0/C:col10/1733811261101/Put/seqid=0 2024-12-10T06:14:22,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741872_1048 (size=12151) 2024-12-10T06:14:22,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811322915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811322936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811322936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/03f7b589ccec49628f15849f74702409 2024-12-10T06:14:23,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/d52eec660ce24f7e9d0d486c3a1883f6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d52eec660ce24f7e9d0d486c3a1883f6 2024-12-10T06:14:23,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d52eec660ce24f7e9d0d486c3a1883f6, entries=200, sequenceid=190, filesize=14.2 K 2024-12-10T06:14:23,166 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T06:14:23,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/740dcc45f6c944f7b3cf11535962e82a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/740dcc45f6c944f7b3cf11535962e82a 2024-12-10T06:14:23,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/740dcc45f6c944f7b3cf11535962e82a, entries=150, sequenceid=190, filesize=11.9 K 2024-12-10T06:14:23,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/03f7b589ccec49628f15849f74702409 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/03f7b589ccec49628f15849f74702409 2024-12-10T06:14:23,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/03f7b589ccec49628f15849f74702409, entries=150, sequenceid=190, filesize=11.9 K 2024-12-10T06:14:23,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 934ms, sequenceid=190, compaction requested=true 2024-12-10T06:14:23,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:23,182 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:23,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:23,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:23,183 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:23,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:23,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:23,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:23,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:23,188 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53658 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:23,188 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/A is initiating minor compaction (all files) 2024-12-10T06:14:23,189 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/A in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:23,189 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/121bf73ad88347d29bcd48e95e127a45, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/e757c74ffad54262b80a6d4012b8d9f9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dfcbcde06bd841ca93d456053803a2b6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d52eec660ce24f7e9d0d486c3a1883f6] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=52.4 K 2024-12-10T06:14:23,190 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:23,190 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/B is initiating minor compaction (all files) 2024-12-10T06:14:23,190 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/B in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:23,190 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/6f2a36cbdeb34ba688b4dfa0d82060af, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/cc93bb07204b4be48be5cb6df204d33e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/35b59f611e324e8ea8e8315d5c130dd5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/740dcc45f6c944f7b3cf11535962e82a] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=47.7 K 2024-12-10T06:14:23,190 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 121bf73ad88347d29bcd48e95e127a45, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733811259020 2024-12-10T06:14:23,191 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f2a36cbdeb34ba688b4dfa0d82060af, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733811259020 2024-12-10T06:14:23,191 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting e757c74ffad54262b80a6d4012b8d9f9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733811259180 2024-12-10T06:14:23,191 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting cc93bb07204b4be48be5cb6df204d33e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733811259198 2024-12-10T06:14:23,192 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfcbcde06bd841ca93d456053803a2b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733811259865 2024-12-10T06:14:23,192 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b59f611e324e8ea8e8315d5c130dd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733811259865 2024-12-10T06:14:23,192 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 740dcc45f6c944f7b3cf11535962e82a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811261101 2024-12-10T06:14:23,192 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting d52eec660ce24f7e9d0d486c3a1883f6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811261077 2024-12-10T06:14:23,209 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:23,210 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/a0f828aa1f2a4278916fbd1757ebf824 is 50, key is test_row_0/A:col10/1733811261101/Put/seqid=0 2024-12-10T06:14:23,212 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#B#compaction#37 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:23,212 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/b0db0263c1c64ea8ac19286f80df854b is 50, key is test_row_0/B:col10/1733811261101/Put/seqid=0 2024-12-10T06:14:23,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741873_1049 (size=12561) 2024-12-10T06:14:23,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741874_1050 (size=12561) 2024-12-10T06:14:23,266 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/b0db0263c1c64ea8ac19286f80df854b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b0db0263c1c64ea8ac19286f80df854b 2024-12-10T06:14:23,274 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/B of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into b0db0263c1c64ea8ac19286f80df854b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:23,274 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:23,274 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/B, priority=12, startTime=1733811263183; duration=0sec 2024-12-10T06:14:23,275 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:23,275 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:B 2024-12-10T06:14:23,275 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:23,277 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:23,277 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/C is initiating minor compaction (all files) 2024-12-10T06:14:23,277 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/C in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:23,277 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8bf7063a0d1e4ac8ad3bf5b93244c00f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0d850dc92d6f45a8877f0ba17477da0a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/7ecfa963fbb1438e83ec2121c3f54789, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/03f7b589ccec49628f15849f74702409] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=47.7 K 2024-12-10T06:14:23,278 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bf7063a0d1e4ac8ad3bf5b93244c00f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733811259020 2024-12-10T06:14:23,278 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d850dc92d6f45a8877f0ba17477da0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733811259198 2024-12-10T06:14:23,279 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ecfa963fbb1438e83ec2121c3f54789, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733811259865 2024-12-10T06:14:23,279 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f7b589ccec49628f15849f74702409, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811261101 2024-12-10T06:14:23,300 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#C#compaction#38 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:23,301 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/8333da3fac0148dd887c1972e1417b28 is 50, key is test_row_0/C:col10/1733811261101/Put/seqid=0 2024-12-10T06:14:23,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741875_1051 (size=12561) 2024-12-10T06:14:23,324 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/8333da3fac0148dd887c1972e1417b28 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8333da3fac0148dd887c1972e1417b28 2024-12-10T06:14:23,332 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/C of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 8333da3fac0148dd887c1972e1417b28(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:23,332 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:23,332 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/C, priority=12, startTime=1733811263183; duration=0sec 2024-12-10T06:14:23,332 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:23,332 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:C 2024-12-10T06:14:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T06:14:23,426 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:23,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-10T06:14:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T06:14:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:23,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T06:14:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:23,430 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:23,433 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:23,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:23,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/f35de441a3124d1e838757ed28f01085 is 50, key is test_row_0/A:col10/1733811263427/Put/seqid=0 2024-12-10T06:14:23,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741876_1052 (size=14541) 2024-12-10T06:14:23,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811323476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811323478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811323480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T06:14:23,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T06:14:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:23,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:23,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:23,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811323585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811323585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811323595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,650 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/a0f828aa1f2a4278916fbd1757ebf824 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a0f828aa1f2a4278916fbd1757ebf824 2024-12-10T06:14:23,658 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/A of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into a0f828aa1f2a4278916fbd1757ebf824(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:23,658 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:23,658 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/A, priority=12, startTime=1733811263182; duration=0sec 2024-12-10T06:14:23,659 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:23,659 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:A 2024-12-10T06:14:23,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T06:14:23,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:23,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:23,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:23,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:23,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T06:14:23,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811323795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:23,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811323795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811323806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:23,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/f35de441a3124d1e838757ed28f01085 2024-12-10T06:14:23,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/239a33ba65a648148f5f24687cb9a42f is 50, key is test_row_0/B:col10/1733811263427/Put/seqid=0 2024-12-10T06:14:23,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741877_1053 (size=12151) 2024-12-10T06:14:23,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T06:14:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:23,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:23,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:23,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:23,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:24,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T06:14:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:24,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T06:14:24,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811324105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811324105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811324116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T06:14:24,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:24,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:24,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/239a33ba65a648148f5f24687cb9a42f 2024-12-10T06:14:24,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/730b1b34ced8426e8c67802bed00bf63 is 50, key is test_row_0/C:col10/1733811263427/Put/seqid=0 2024-12-10T06:14:24,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741878_1054 (size=12151) 2024-12-10T06:14:24,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/730b1b34ced8426e8c67802bed00bf63 2024-12-10T06:14:24,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/f35de441a3124d1e838757ed28f01085 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f35de441a3124d1e838757ed28f01085 2024-12-10T06:14:24,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f35de441a3124d1e838757ed28f01085, entries=200, sequenceid=206, filesize=14.2 K 2024-12-10T06:14:24,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/239a33ba65a648148f5f24687cb9a42f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/239a33ba65a648148f5f24687cb9a42f 2024-12-10T06:14:24,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/239a33ba65a648148f5f24687cb9a42f, entries=150, sequenceid=206, filesize=11.9 K 2024-12-10T06:14:24,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/730b1b34ced8426e8c67802bed00bf63 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/730b1b34ced8426e8c67802bed00bf63 2024-12-10T06:14:24,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/730b1b34ced8426e8c67802bed00bf63, entries=150, sequenceid=206, filesize=11.9 K 2024-12-10T06:14:24,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 887ms, sequenceid=206, compaction requested=false 2024-12-10T06:14:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:24,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T06:14:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:24,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T06:14:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/9f9f04d718b84d5bad84a68a630bd96d is 50, key is test_row_0/A:col10/1733811263474/Put/seqid=0 2024-12-10T06:14:24,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741879_1055 (size=12151) 2024-12-10T06:14:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T06:14:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:24,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:24,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811324645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811324647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811324648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811324755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811324765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811324765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,779 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/9f9f04d718b84d5bad84a68a630bd96d 2024-12-10T06:14:24,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/9fc53ee2289e4d18a25c2edb4f65e64b is 50, key is test_row_0/B:col10/1733811263474/Put/seqid=0 2024-12-10T06:14:24,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741880_1056 (size=12151) 2024-12-10T06:14:24,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811324966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811324975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:24,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811324975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:25,197 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/9fc53ee2289e4d18a25c2edb4f65e64b 2024-12-10T06:14:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/87fb15ae0d194bfe92f0928c7b5f04cd is 50, key is test_row_0/C:col10/1733811263474/Put/seqid=0 2024-12-10T06:14:25,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741881_1057 (size=12151) 2024-12-10T06:14:25,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811325275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:25,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811325286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811325286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T06:14:25,616 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/87fb15ae0d194bfe92f0928c7b5f04cd 2024-12-10T06:14:25,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/9f9f04d718b84d5bad84a68a630bd96d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9f9f04d718b84d5bad84a68a630bd96d 2024-12-10T06:14:25,630 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9f9f04d718b84d5bad84a68a630bd96d, entries=150, sequenceid=225, filesize=11.9 K 2024-12-10T06:14:25,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/9fc53ee2289e4d18a25c2edb4f65e64b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9fc53ee2289e4d18a25c2edb4f65e64b 2024-12-10T06:14:25,638 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9fc53ee2289e4d18a25c2edb4f65e64b, entries=150, sequenceid=225, filesize=11.9 K 2024-12-10T06:14:25,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/87fb15ae0d194bfe92f0928c7b5f04cd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/87fb15ae0d194bfe92f0928c7b5f04cd 2024-12-10T06:14:25,646 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/87fb15ae0d194bfe92f0928c7b5f04cd, entries=150, sequenceid=225, filesize=11.9 K 2024-12-10T06:14:25,647 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 1288ms, sequenceid=225, compaction requested=true 2024-12-10T06:14:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-10T06:14:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-10T06:14:25,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-10T06:14:25,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2160 sec 2024-12-10T06:14:25,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.2270 sec 2024-12-10T06:14:25,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:14:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:25,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:25,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:25,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:25,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:25,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:25,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:25,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/47fedb90c0c94d1ea35e52bdc812a968 is 50, key is test_row_0/A:col10/1733811265797/Put/seqid=0 2024-12-10T06:14:25,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741882_1058 (size=12151) 2024-12-10T06:14:25,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811325872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811325872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811325872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811325985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811325986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811325986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811326195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811326196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811326196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/47fedb90c0c94d1ea35e52bdc812a968 2024-12-10T06:14:26,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/92f06fefaa3f469c8baaaa6e357cc4d5 is 50, key is test_row_0/B:col10/1733811265797/Put/seqid=0 2024-12-10T06:14:26,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741883_1059 (size=12151) 2024-12-10T06:14:26,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/92f06fefaa3f469c8baaaa6e357cc4d5 2024-12-10T06:14:26,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/da2e590ef1b94723af1466bab839080e is 50, key is test_row_0/C:col10/1733811265797/Put/seqid=0 2024-12-10T06:14:26,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741884_1060 (size=12151) 2024-12-10T06:14:26,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/da2e590ef1b94723af1466bab839080e 2024-12-10T06:14:26,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/47fedb90c0c94d1ea35e52bdc812a968 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/47fedb90c0c94d1ea35e52bdc812a968 2024-12-10T06:14:26,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/47fedb90c0c94d1ea35e52bdc812a968, entries=150, sequenceid=242, filesize=11.9 K 2024-12-10T06:14:26,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/92f06fefaa3f469c8baaaa6e357cc4d5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/92f06fefaa3f469c8baaaa6e357cc4d5 2024-12-10T06:14:26,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/92f06fefaa3f469c8baaaa6e357cc4d5, entries=150, sequenceid=242, filesize=11.9 K 2024-12-10T06:14:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/da2e590ef1b94723af1466bab839080e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/da2e590ef1b94723af1466bab839080e 2024-12-10T06:14:26,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/da2e590ef1b94723af1466bab839080e, entries=150, sequenceid=242, filesize=11.9 K 2024-12-10T06:14:26,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 484ms, sequenceid=242, compaction requested=true 2024-12-10T06:14:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:26,283 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:26,283 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:26,286 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:26,286 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:26,286 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/A is initiating minor compaction (all files) 2024-12-10T06:14:26,286 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/B is initiating minor compaction (all files) 2024-12-10T06:14:26,286 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/A in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:26,286 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/B in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:26,286 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b0db0263c1c64ea8ac19286f80df854b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/239a33ba65a648148f5f24687cb9a42f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9fc53ee2289e4d18a25c2edb4f65e64b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/92f06fefaa3f469c8baaaa6e357cc4d5] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=47.9 K 2024-12-10T06:14:26,286 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a0f828aa1f2a4278916fbd1757ebf824, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f35de441a3124d1e838757ed28f01085, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9f9f04d718b84d5bad84a68a630bd96d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/47fedb90c0c94d1ea35e52bdc812a968] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=50.2 K 2024-12-10T06:14:26,287 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b0db0263c1c64ea8ac19286f80df854b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811261101 2024-12-10T06:14:26,287 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0f828aa1f2a4278916fbd1757ebf824, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811261101 2024-12-10T06:14:26,287 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 239a33ba65a648148f5f24687cb9a42f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733811262276 2024-12-10T06:14:26,287 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f35de441a3124d1e838757ed28f01085, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733811262269 2024-12-10T06:14:26,288 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc53ee2289e4d18a25c2edb4f65e64b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733811263468 2024-12-10T06:14:26,288 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f9f04d718b84d5bad84a68a630bd96d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733811263468 2024-12-10T06:14:26,288 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f06fefaa3f469c8baaaa6e357cc4d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733811264644 2024-12-10T06:14:26,288 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47fedb90c0c94d1ea35e52bdc812a968, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733811264644 2024-12-10T06:14:26,301 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#B#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:26,301 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/f458f54d53f640298effde81fc60e844 is 50, key is test_row_0/B:col10/1733811265797/Put/seqid=0 2024-12-10T06:14:26,302 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:26,305 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/c10ea3432c5b4933b036045c076ba011 is 50, key is test_row_0/A:col10/1733811265797/Put/seqid=0 2024-12-10T06:14:26,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741885_1061 (size=12697) 2024-12-10T06:14:26,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741886_1062 (size=12697) 2024-12-10T06:14:26,320 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/c10ea3432c5b4933b036045c076ba011 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c10ea3432c5b4933b036045c076ba011 2024-12-10T06:14:26,331 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/A of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into c10ea3432c5b4933b036045c076ba011(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:26,331 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:26,331 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/A, priority=12, startTime=1733811266283; duration=0sec 2024-12-10T06:14:26,331 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:26,331 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:A 2024-12-10T06:14:26,331 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:26,334 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:26,335 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/C is initiating minor compaction (all files) 2024-12-10T06:14:26,335 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/C in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:26,335 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8333da3fac0148dd887c1972e1417b28, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/730b1b34ced8426e8c67802bed00bf63, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/87fb15ae0d194bfe92f0928c7b5f04cd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/da2e590ef1b94723af1466bab839080e] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=47.9 K 2024-12-10T06:14:26,336 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8333da3fac0148dd887c1972e1417b28, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811261101 2024-12-10T06:14:26,338 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 730b1b34ced8426e8c67802bed00bf63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733811262276 2024-12-10T06:14:26,339 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87fb15ae0d194bfe92f0928c7b5f04cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733811263468 2024-12-10T06:14:26,340 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting da2e590ef1b94723af1466bab839080e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733811264644 2024-12-10T06:14:26,353 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:26,354 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/725c892df10e47ce9c26bc4c93b525dd is 50, key is test_row_0/C:col10/1733811265797/Put/seqid=0 2024-12-10T06:14:26,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741887_1063 (size=12697) 2024-12-10T06:14:26,372 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/725c892df10e47ce9c26bc4c93b525dd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/725c892df10e47ce9c26bc4c93b525dd 2024-12-10T06:14:26,380 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/C of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 725c892df10e47ce9c26bc4c93b525dd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:26,381 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:26,381 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/C, priority=12, startTime=1733811266283; duration=0sec 2024-12-10T06:14:26,381 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:26,381 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:C 2024-12-10T06:14:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:26,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T06:14:26,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:26,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:26,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:26,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:26,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:26,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:26,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/3eaa5ac0d4bc42b0bbeb6e64febf192c is 50, key is test_row_0/A:col10/1733811265818/Put/seqid=0 2024-12-10T06:14:26,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741888_1064 (size=14691) 2024-12-10T06:14:26,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811326540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811326542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811326543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50384 deadline: 1733811326595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,602 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 91cf7a8c24ad1cfeb344be5ce72aa8a5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50370 deadline: 1733811326625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,630 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 91cf7a8c24ad1cfeb344be5ce72aa8a5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811326645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811326670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811326670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,714 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/f458f54d53f640298effde81fc60e844 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/f458f54d53f640298effde81fc60e844 2024-12-10T06:14:26,720 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/B of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into f458f54d53f640298effde81fc60e844(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:26,720 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:26,720 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/B, priority=12, startTime=1733811266283; duration=0sec 2024-12-10T06:14:26,721 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:26,721 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:B 2024-12-10T06:14:26,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811326856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811326875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811326875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:26,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/3eaa5ac0d4bc42b0bbeb6e64febf192c 2024-12-10T06:14:26,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/5b65b55e56ad4566b0687ca6e8016034 is 50, key is test_row_0/B:col10/1733811265818/Put/seqid=0 2024-12-10T06:14:26,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741889_1065 (size=12251) 2024-12-10T06:14:27,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811327176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:27,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811327185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:27,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811327195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:27,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/5b65b55e56ad4566b0687ca6e8016034 2024-12-10T06:14:27,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/40299e4f32f143008ddb98c5a9baf303 is 50, key is test_row_0/C:col10/1733811265818/Put/seqid=0 2024-12-10T06:14:27,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741890_1066 (size=12251) 2024-12-10T06:14:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T06:14:27,596 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:27,596 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-10T06:14:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T06:14:27,601 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:27,602 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:27,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:27,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811327685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:27,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811327695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T06:14:27,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811327706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:27,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T06:14:27,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:27,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:27,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:27,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:27,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:27,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/40299e4f32f143008ddb98c5a9baf303 2024-12-10T06:14:27,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/3eaa5ac0d4bc42b0bbeb6e64febf192c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/3eaa5ac0d4bc42b0bbeb6e64febf192c 2024-12-10T06:14:27,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/3eaa5ac0d4bc42b0bbeb6e64febf192c, entries=200, sequenceid=262, filesize=14.3 K 2024-12-10T06:14:27,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/5b65b55e56ad4566b0687ca6e8016034 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/5b65b55e56ad4566b0687ca6e8016034 2024-12-10T06:14:27,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/5b65b55e56ad4566b0687ca6e8016034, entries=150, sequenceid=262, filesize=12.0 K 2024-12-10T06:14:27,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/40299e4f32f143008ddb98c5a9baf303 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/40299e4f32f143008ddb98c5a9baf303 2024-12-10T06:14:27,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/40299e4f32f143008ddb98c5a9baf303, entries=150, sequenceid=262, filesize=12.0 K 2024-12-10T06:14:27,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 1312ms, sequenceid=262, compaction requested=false 2024-12-10T06:14:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:27,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T06:14:27,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:27,909 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T06:14:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T06:14:27,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/6c25c5f8f9e84f8aba19a2506b0ae738 is 50, key is test_row_0/A:col10/1733811266539/Put/seqid=0 2024-12-10T06:14:27,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741891_1067 (size=12301) 2024-12-10T06:14:27,922 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/6c25c5f8f9e84f8aba19a2506b0ae738 2024-12-10T06:14:27,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/b4e0d841fbd8471f931e06cf8f88eca0 is 50, key is test_row_0/B:col10/1733811266539/Put/seqid=0 2024-12-10T06:14:27,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741892_1068 (size=12301) 2024-12-10T06:14:27,945 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/b4e0d841fbd8471f931e06cf8f88eca0 2024-12-10T06:14:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/b003496f69e34f21ba4a1eec9263591d is 50, key is test_row_0/C:col10/1733811266539/Put/seqid=0 2024-12-10T06:14:27,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741893_1069 (size=12301) 2024-12-10T06:14:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T06:14:28,368 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/b003496f69e34f21ba4a1eec9263591d 2024-12-10T06:14:28,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/6c25c5f8f9e84f8aba19a2506b0ae738 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/6c25c5f8f9e84f8aba19a2506b0ae738 2024-12-10T06:14:28,381 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/6c25c5f8f9e84f8aba19a2506b0ae738, entries=150, sequenceid=278, filesize=12.0 K 2024-12-10T06:14:28,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/b4e0d841fbd8471f931e06cf8f88eca0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b4e0d841fbd8471f931e06cf8f88eca0 2024-12-10T06:14:28,388 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b4e0d841fbd8471f931e06cf8f88eca0, entries=150, sequenceid=278, filesize=12.0 K 2024-12-10T06:14:28,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/b003496f69e34f21ba4a1eec9263591d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/b003496f69e34f21ba4a1eec9263591d 2024-12-10T06:14:28,396 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/b003496f69e34f21ba4a1eec9263591d, entries=150, sequenceid=278, filesize=12.0 K 2024-12-10T06:14:28,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 490ms, sequenceid=278, compaction requested=true 2024-12-10T06:14:28,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:28,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:28,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-10T06:14:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-10T06:14:28,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-10T06:14:28,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 801 msec 2024-12-10T06:14:28,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 812 msec 2024-12-10T06:14:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:28,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T06:14:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T06:14:28,735 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:28,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/a85ab9b97d0e48528a68bf88a36cab32 is 50, key is test_row_0/A:col10/1733811268715/Put/seqid=0 2024-12-10T06:14:28,735 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-10T06:14:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T06:14:28,740 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:28,741 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:28,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:28,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741894_1070 (size=17181) 2024-12-10T06:14:28,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/a85ab9b97d0e48528a68bf88a36cab32 2024-12-10T06:14:28,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/63a67bc4a35f425a98476249e0e5cda6 is 50, key is test_row_0/B:col10/1733811268715/Put/seqid=0 2024-12-10T06:14:28,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741895_1071 (size=12301) 2024-12-10T06:14:28,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/63a67bc4a35f425a98476249e0e5cda6 2024-12-10T06:14:28,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/c643d712a77740f9a64b6ba1255b7dc1 is 50, key is test_row_0/C:col10/1733811268715/Put/seqid=0 2024-12-10T06:14:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T06:14:28,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811328833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:28,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741896_1072 (size=12301) 2024-12-10T06:14:28,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811268779, max timestamp=1733811268837Num uniques 859; 2024-12-10T06:14:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:28,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:14:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:28,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811268779, max timestamp=1733811268837Num uniques 859; 2024-12-10T06:14:28,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:14:28,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811268779, max timestamp=1733811268837Num uniques 859; 2024-12-10T06:14:28,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:14:28,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T06:14:28,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:28,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:28,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:28,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811328922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:28,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811328925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:28,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811328955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811329035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811329045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T06:14:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:29,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T06:14:29,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811329175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T06:14:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:29,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:29,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811329245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/c643d712a77740f9a64b6ba1255b7dc1 2024-12-10T06:14:29,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/a85ab9b97d0e48528a68bf88a36cab32 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a85ab9b97d0e48528a68bf88a36cab32 2024-12-10T06:14:29,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a85ab9b97d0e48528a68bf88a36cab32, entries=250, sequenceid=289, filesize=16.8 K 2024-12-10T06:14:29,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811329255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/63a67bc4a35f425a98476249e0e5cda6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/63a67bc4a35f425a98476249e0e5cda6 2024-12-10T06:14:29,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/63a67bc4a35f425a98476249e0e5cda6, entries=150, sequenceid=289, filesize=12.0 K 2024-12-10T06:14:29,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/c643d712a77740f9a64b6ba1255b7dc1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/c643d712a77740f9a64b6ba1255b7dc1 2024-12-10T06:14:29,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/c643d712a77740f9a64b6ba1255b7dc1, entries=150, sequenceid=289, filesize=12.0 K 2024-12-10T06:14:29,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 556ms, sequenceid=289, compaction requested=true 2024-12-10T06:14:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:29,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:29,283 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:29,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:29,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:29,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T06:14:29,283 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:29,284 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:29,284 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/A is initiating minor compaction (all files) 2024-12-10T06:14:29,284 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/A in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:29,285 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c10ea3432c5b4933b036045c076ba011, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/3eaa5ac0d4bc42b0bbeb6e64febf192c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/6c25c5f8f9e84f8aba19a2506b0ae738, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a85ab9b97d0e48528a68bf88a36cab32] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=55.5 K 2024-12-10T06:14:29,285 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:29,286 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/B is initiating minor compaction (all files) 2024-12-10T06:14:29,286 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c10ea3432c5b4933b036045c076ba011, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733811264644 2024-12-10T06:14:29,286 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/B in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:29,286 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/f458f54d53f640298effde81fc60e844, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/5b65b55e56ad4566b0687ca6e8016034, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b4e0d841fbd8471f931e06cf8f88eca0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/63a67bc4a35f425a98476249e0e5cda6] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=48.4 K 2024-12-10T06:14:29,286 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eaa5ac0d4bc42b0bbeb6e64febf192c, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733811265818 2024-12-10T06:14:29,286 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting f458f54d53f640298effde81fc60e844, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733811264644 2024-12-10T06:14:29,287 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c25c5f8f9e84f8aba19a2506b0ae738, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733811266535 2024-12-10T06:14:29,287 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b65b55e56ad4566b0687ca6e8016034, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733811265818 2024-12-10T06:14:29,287 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting a85ab9b97d0e48528a68bf88a36cab32, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733811268710 2024-12-10T06:14:29,288 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b4e0d841fbd8471f931e06cf8f88eca0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733811266535 2024-12-10T06:14:29,288 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 63a67bc4a35f425a98476249e0e5cda6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733811268715 2024-12-10T06:14:29,303 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:29,304 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/090e5b35f74b47ad99759b153b256f7e is 50, key is test_row_0/A:col10/1733811268715/Put/seqid=0 2024-12-10T06:14:29,307 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:29,309 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/9ff01d0a46304b419eb1bc67c8c966d9 is 50, key is test_row_0/B:col10/1733811268715/Put/seqid=0 2024-12-10T06:14:29,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741898_1074 (size=12983) 2024-12-10T06:14:29,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741897_1073 (size=12983) 2024-12-10T06:14:29,329 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/090e5b35f74b47ad99759b153b256f7e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/090e5b35f74b47ad99759b153b256f7e 2024-12-10T06:14:29,338 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/A of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 090e5b35f74b47ad99759b153b256f7e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:29,338 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:29,338 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/A, priority=12, startTime=1733811269282; duration=0sec 2024-12-10T06:14:29,338 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:29,338 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:A 2024-12-10T06:14:29,338 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:29,340 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:29,340 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/C is initiating minor compaction (all files) 2024-12-10T06:14:29,340 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/C in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:29,340 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/725c892df10e47ce9c26bc4c93b525dd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/40299e4f32f143008ddb98c5a9baf303, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/b003496f69e34f21ba4a1eec9263591d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/c643d712a77740f9a64b6ba1255b7dc1] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=48.4 K 2024-12-10T06:14:29,341 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725c892df10e47ce9c26bc4c93b525dd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733811264644 2024-12-10T06:14:29,341 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40299e4f32f143008ddb98c5a9baf303, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733811265818 2024-12-10T06:14:29,342 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting b003496f69e34f21ba4a1eec9263591d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733811266535 2024-12-10T06:14:29,343 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c643d712a77740f9a64b6ba1255b7dc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733811268715 2024-12-10T06:14:29,354 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:29,355 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/6404f88eb2044287913651a9fd281778 is 50, key is test_row_0/C:col10/1733811268715/Put/seqid=0 2024-12-10T06:14:29,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T06:14:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:29,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-10T06:14:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:29,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741899_1075 (size=12983) 2024-12-10T06:14:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T06:14:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/c9ef47dbbebc454eb30d8a5eb3cdd417 is 50, key is test_row_0/A:col10/1733811268920/Put/seqid=0 2024-12-10T06:14:29,374 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/6404f88eb2044287913651a9fd281778 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/6404f88eb2044287913651a9fd281778 2024-12-10T06:14:29,384 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/C of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 6404f88eb2044287913651a9fd281778(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:29,384 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:29,384 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/C, priority=12, startTime=1733811269283; duration=0sec 2024-12-10T06:14:29,384 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:29,384 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:C 2024-12-10T06:14:29,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741900_1076 (size=12301) 2024-12-10T06:14:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:29,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:29,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811329526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811329555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811329575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811329635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,726 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/9ff01d0a46304b419eb1bc67c8c966d9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9ff01d0a46304b419eb1bc67c8c966d9 2024-12-10T06:14:29,733 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/B of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 9ff01d0a46304b419eb1bc67c8c966d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:29,733 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:29,734 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/B, priority=12, startTime=1733811269283; duration=0sec 2024-12-10T06:14:29,734 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:29,734 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:B 2024-12-10T06:14:29,789 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/c9ef47dbbebc454eb30d8a5eb3cdd417 2024-12-10T06:14:29,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8ae8125cda3a4bb4880cfc5ef09cc897 is 50, key is test_row_0/B:col10/1733811268920/Put/seqid=0 2024-12-10T06:14:29,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741901_1077 (size=12301) 2024-12-10T06:14:29,809 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8ae8125cda3a4bb4880cfc5ef09cc897 2024-12-10T06:14:29,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/d7950b91a13b4d2695ed57727e6749a6 is 50, key is test_row_0/C:col10/1733811268920/Put/seqid=0 2024-12-10T06:14:29,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741902_1078 (size=12301) 2024-12-10T06:14:29,835 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/d7950b91a13b4d2695ed57727e6749a6 2024-12-10T06:14:29,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/c9ef47dbbebc454eb30d8a5eb3cdd417 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c9ef47dbbebc454eb30d8a5eb3cdd417 2024-12-10T06:14:29,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811329846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:29,851 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c9ef47dbbebc454eb30d8a5eb3cdd417, entries=150, sequenceid=326, filesize=12.0 K 2024-12-10T06:14:29,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8ae8125cda3a4bb4880cfc5ef09cc897 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8ae8125cda3a4bb4880cfc5ef09cc897 2024-12-10T06:14:29,860 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8ae8125cda3a4bb4880cfc5ef09cc897, entries=150, sequenceid=326, filesize=12.0 K 2024-12-10T06:14:29,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/d7950b91a13b4d2695ed57727e6749a6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/d7950b91a13b4d2695ed57727e6749a6 2024-12-10T06:14:29,874 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/d7950b91a13b4d2695ed57727e6749a6, entries=150, sequenceid=326, filesize=12.0 K 2024-12-10T06:14:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T06:14:29,876 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 518ms, sequenceid=326, compaction requested=false 2024-12-10T06:14:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-10T06:14:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-10T06:14:29,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-10T06:14:29,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1380 sec 2024-12-10T06:14:29,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1490 sec 2024-12-10T06:14:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:30,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T06:14:30,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:30,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:30,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:30,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:30,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:30,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:30,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/4747112d5fdd4546b9ef6b4e43bdbd64 is 50, key is test_row_0/A:col10/1733811269524/Put/seqid=0 2024-12-10T06:14:30,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741903_1079 (size=14741) 2024-12-10T06:14:30,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/4747112d5fdd4546b9ef6b4e43bdbd64 2024-12-10T06:14:30,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/395aa80536ec478284677c7aa75b3876 is 50, key is test_row_0/B:col10/1733811269524/Put/seqid=0 2024-12-10T06:14:30,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741904_1080 (size=12301) 2024-12-10T06:14:30,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/395aa80536ec478284677c7aa75b3876 2024-12-10T06:14:30,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0151f8b4dd1345129c9bb780c1036b79 is 50, key is test_row_0/C:col10/1733811269524/Put/seqid=0 2024-12-10T06:14:30,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811330143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811330144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741905_1081 (size=12301) 2024-12-10T06:14:30,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811330155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811330255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811330255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811330465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811330465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0151f8b4dd1345129c9bb780c1036b79 2024-12-10T06:14:30,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/4747112d5fdd4546b9ef6b4e43bdbd64 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4747112d5fdd4546b9ef6b4e43bdbd64 2024-12-10T06:14:30,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4747112d5fdd4546b9ef6b4e43bdbd64, entries=200, sequenceid=340, filesize=14.4 K 2024-12-10T06:14:30,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/395aa80536ec478284677c7aa75b3876 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/395aa80536ec478284677c7aa75b3876 2024-12-10T06:14:30,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/395aa80536ec478284677c7aa75b3876, entries=150, sequenceid=340, filesize=12.0 K 2024-12-10T06:14:30,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0151f8b4dd1345129c9bb780c1036b79 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0151f8b4dd1345129c9bb780c1036b79 2024-12-10T06:14:30,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0151f8b4dd1345129c9bb780c1036b79, entries=150, sequenceid=340, filesize=12.0 K 2024-12-10T06:14:30,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 512ms, sequenceid=340, compaction requested=true 2024-12-10T06:14:30,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:30,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:30,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:30,581 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:30,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:30,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:30,582 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:30,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:30,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:30,583 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:30,583 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:30,584 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/A is initiating minor compaction (all files) 2024-12-10T06:14:30,584 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/B is initiating minor compaction (all files) 2024-12-10T06:14:30,584 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/B in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:30,584 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/A in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:30,584 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/090e5b35f74b47ad99759b153b256f7e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c9ef47dbbebc454eb30d8a5eb3cdd417, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4747112d5fdd4546b9ef6b4e43bdbd64] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=39.1 K 2024-12-10T06:14:30,584 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9ff01d0a46304b419eb1bc67c8c966d9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8ae8125cda3a4bb4880cfc5ef09cc897, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/395aa80536ec478284677c7aa75b3876] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=36.7 K 2024-12-10T06:14:30,584 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ff01d0a46304b419eb1bc67c8c966d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733811268715 2024-12-10T06:14:30,584 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 090e5b35f74b47ad99759b153b256f7e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733811268715 2024-12-10T06:14:30,585 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae8125cda3a4bb4880cfc5ef09cc897, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733811268918 2024-12-10T06:14:30,585 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9ef47dbbebc454eb30d8a5eb3cdd417, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733811268918 2024-12-10T06:14:30,585 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4747112d5fdd4546b9ef6b4e43bdbd64, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733811269506 2024-12-10T06:14:30,585 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 395aa80536ec478284677c7aa75b3876, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733811269520 2024-12-10T06:14:30,599 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:30,599 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:30,600 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/ec4544ee74094c6fa93502dc6c28dfeb is 50, key is test_row_0/A:col10/1733811269524/Put/seqid=0 2024-12-10T06:14:30,600 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8719aba1a6eb4bbf85aedc89f57d325c is 50, key is test_row_0/B:col10/1733811269524/Put/seqid=0 2024-12-10T06:14:30,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741907_1083 (size=13085) 2024-12-10T06:14:30,614 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/ec4544ee74094c6fa93502dc6c28dfeb as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/ec4544ee74094c6fa93502dc6c28dfeb 2024-12-10T06:14:30,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741906_1082 (size=13085) 2024-12-10T06:14:30,622 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/A of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into ec4544ee74094c6fa93502dc6c28dfeb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:30,622 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:30,622 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/A, priority=13, startTime=1733811270581; duration=0sec 2024-12-10T06:14:30,623 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:30,623 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:A 2024-12-10T06:14:30,623 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:30,626 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:30,626 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/C is initiating minor compaction (all files) 2024-12-10T06:14:30,627 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/C in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:30,627 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/6404f88eb2044287913651a9fd281778, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/d7950b91a13b4d2695ed57727e6749a6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0151f8b4dd1345129c9bb780c1036b79] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=36.7 K 2024-12-10T06:14:30,627 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6404f88eb2044287913651a9fd281778, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733811268715 2024-12-10T06:14:30,628 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7950b91a13b4d2695ed57727e6749a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733811268918 2024-12-10T06:14:30,628 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/8719aba1a6eb4bbf85aedc89f57d325c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8719aba1a6eb4bbf85aedc89f57d325c 2024-12-10T06:14:30,628 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0151f8b4dd1345129c9bb780c1036b79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733811269520 2024-12-10T06:14:30,635 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/B of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 8719aba1a6eb4bbf85aedc89f57d325c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:30,636 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:30,636 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/B, priority=13, startTime=1733811270581; duration=0sec 2024-12-10T06:14:30,636 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:30,636 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:B 2024-12-10T06:14:30,643 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:30,644 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0c5ddd38726646e18c67f89cd6ab66af is 50, key is test_row_0/C:col10/1733811269524/Put/seqid=0 2024-12-10T06:14:30,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741908_1084 (size=13085) 2024-12-10T06:14:30,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T06:14:30,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:30,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:30,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:30,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:30,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:30,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:30,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/8e9324d8491445198b5bed0ac295315c is 50, key is test_row_0/A:col10/1733811270142/Put/seqid=0 2024-12-10T06:14:30,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741909_1085 (size=12301) 2024-12-10T06:14:30,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/8e9324d8491445198b5bed0ac295315c 2024-12-10T06:14:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811330754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/0e335dcbd24342bb9dfb5fe02ad078a5 is 50, key is test_row_0/B:col10/1733811270142/Put/seqid=0 2024-12-10T06:14:30,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811330785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811330785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:30,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741910_1086 (size=12301) 2024-12-10T06:14:30,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/0e335dcbd24342bb9dfb5fe02ad078a5 2024-12-10T06:14:30,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/095d8aa3c371493f9cc93e5271ed977f is 50, key is test_row_0/C:col10/1733811270142/Put/seqid=0 2024-12-10T06:14:30,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741911_1087 (size=12301) 2024-12-10T06:14:30,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/095d8aa3c371493f9cc93e5271ed977f 2024-12-10T06:14:30,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/8e9324d8491445198b5bed0ac295315c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/8e9324d8491445198b5bed0ac295315c 2024-12-10T06:14:30,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/8e9324d8491445198b5bed0ac295315c, entries=150, sequenceid=363, filesize=12.0 K 2024-12-10T06:14:30,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/0e335dcbd24342bb9dfb5fe02ad078a5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/0e335dcbd24342bb9dfb5fe02ad078a5 2024-12-10T06:14:30,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/0e335dcbd24342bb9dfb5fe02ad078a5, entries=150, sequenceid=363, filesize=12.0 K 2024-12-10T06:14:30,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/095d8aa3c371493f9cc93e5271ed977f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/095d8aa3c371493f9cc93e5271ed977f 2024-12-10T06:14:30,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/095d8aa3c371493f9cc93e5271ed977f, entries=150, sequenceid=363, filesize=12.0 K 2024-12-10T06:14:30,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 166ms, sequenceid=363, compaction requested=false 2024-12-10T06:14:30,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:30,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T06:14:30,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:30,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:30,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:30,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:30,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:30,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:30,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/c8805fb7f3ed4cb2bc5319d1a4e3b5ba is 50, key is test_row_0/A:col10/1733811270751/Put/seqid=0 2024-12-10T06:14:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T06:14:30,886 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:30,886 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:30,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741912_1088 (size=14741) 2024-12-10T06:14:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-10T06:14:30,889 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T06:14:30,891 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:30,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:30,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/c8805fb7f3ed4cb2bc5319d1a4e3b5ba 2024-12-10T06:14:30,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/fb9c26a0e8914274a2cd2f01e7d3eabe is 50, key is test_row_0/B:col10/1733811270751/Put/seqid=0 2024-12-10T06:14:30,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741913_1089 (size=12301) 2024-12-10T06:14:30,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/fb9c26a0e8914274a2cd2f01e7d3eabe 2024-12-10T06:14:30,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/727a22c84a064df39936dfaf6669f8f1 is 50, key is test_row_0/C:col10/1733811270751/Put/seqid=0 2024-12-10T06:14:30,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741914_1090 (size=12301) 2024-12-10T06:14:30,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/727a22c84a064df39936dfaf6669f8f1 2024-12-10T06:14:30,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/c8805fb7f3ed4cb2bc5319d1a4e3b5ba as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c8805fb7f3ed4cb2bc5319d1a4e3b5ba 2024-12-10T06:14:30,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c8805fb7f3ed4cb2bc5319d1a4e3b5ba, entries=200, sequenceid=375, filesize=14.4 K 2024-12-10T06:14:30,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/fb9c26a0e8914274a2cd2f01e7d3eabe as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/fb9c26a0e8914274a2cd2f01e7d3eabe 2024-12-10T06:14:30,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/fb9c26a0e8914274a2cd2f01e7d3eabe, entries=150, sequenceid=375, filesize=12.0 K 2024-12-10T06:14:30,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/727a22c84a064df39936dfaf6669f8f1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/727a22c84a064df39936dfaf6669f8f1 2024-12-10T06:14:30,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/727a22c84a064df39936dfaf6669f8f1, entries=150, sequenceid=375, filesize=12.0 K 2024-12-10T06:14:30,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=80.51 KB/82440 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 99ms, sequenceid=375, compaction requested=true 2024-12-10T06:14:30,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:30,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:A, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:30,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:30,968 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:30,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:B, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T06:14:30,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:14:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:30,970 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:30,970 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/A is initiating minor compaction (all files) 2024-12-10T06:14:30,970 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/A in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:30,970 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/ec4544ee74094c6fa93502dc6c28dfeb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/8e9324d8491445198b5bed0ac295315c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c8805fb7f3ed4cb2bc5319d1a4e3b5ba] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=39.2 K 2024-12-10T06:14:30,971 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4544ee74094c6fa93502dc6c28dfeb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733811269520 2024-12-10T06:14:30,971 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e9324d8491445198b5bed0ac295315c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733811270130 2024-12-10T06:14:30,972 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting c8805fb7f3ed4cb2bc5319d1a4e3b5ba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733811270718 2024-12-10T06:14:30,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/cc4977f36c1f48cfb9a04f19c55f2fb8 is 50, key is test_row_0/A:col10/1733811270956/Put/seqid=0 2024-12-10T06:14:30,994 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T06:14:30,995 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/9e1e2b309d7649a09c6810aee81bbcad is 50, key is test_row_0/A:col10/1733811270751/Put/seqid=0 2024-12-10T06:14:31,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741916_1092 (size=13187) 2024-12-10T06:14:31,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741915_1091 (size=14741) 2024-12-10T06:14:31,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/cc4977f36c1f48cfb9a04f19c55f2fb8 2024-12-10T06:14:31,023 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/9e1e2b309d7649a09c6810aee81bbcad as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9e1e2b309d7649a09c6810aee81bbcad 2024-12-10T06:14:31,035 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/A of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 9e1e2b309d7649a09c6810aee81bbcad(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:31,035 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:31,035 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/A, priority=13, startTime=1733811270968; duration=0sec 2024-12-10T06:14:31,035 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T06:14:31,035 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:A 2024-12-10T06:14:31,035 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-12-10T06:14:31,038 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T06:14:31,039 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T06:14:31,039 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. because compaction request was cancelled 2024-12-10T06:14:31,039 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:C 2024-12-10T06:14:31,039 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:31,040 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:31,041 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/B is initiating minor compaction (all files) 2024-12-10T06:14:31,041 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/B in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:31,041 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8719aba1a6eb4bbf85aedc89f57d325c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/0e335dcbd24342bb9dfb5fe02ad078a5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/fb9c26a0e8914274a2cd2f01e7d3eabe] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=36.8 K 2024-12-10T06:14:31,041 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8719aba1a6eb4bbf85aedc89f57d325c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733811269520 2024-12-10T06:14:31,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/eb54e0924dce4c5bb0443d00c27188e1 is 50, key is test_row_0/B:col10/1733811270956/Put/seqid=0 2024-12-10T06:14:31,042 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e335dcbd24342bb9dfb5fe02ad078a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733811270130 2024-12-10T06:14:31,042 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting fb9c26a0e8914274a2cd2f01e7d3eabe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733811270718 2024-12-10T06:14:31,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T06:14:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:31,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741917_1093 (size=12301) 2024-12-10T06:14:31,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:31,053 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:31,054 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/278ff28185bb47c59a87649fd9631c1b is 50, key is test_row_0/B:col10/1733811270751/Put/seqid=0 2024-12-10T06:14:31,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741918_1094 (size=13187) 2024-12-10T06:14:31,067 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/0c5ddd38726646e18c67f89cd6ab66af as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0c5ddd38726646e18c67f89cd6ab66af 2024-12-10T06:14:31,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811331070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:31,090 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/C of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 0c5ddd38726646e18c67f89cd6ab66af(size=12.8 K), total size for store is 36.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:31,090 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:31,090 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/C, priority=13, startTime=1733811270582; duration=0sec 2024-12-10T06:14:31,091 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:31,091 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:C 2024-12-10T06:14:31,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811331185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:31,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T06:14:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:31,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T06:14:31,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811331295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:31,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811331295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:31,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T06:14:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:31,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811331405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:31,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/eb54e0924dce4c5bb0443d00c27188e1 2024-12-10T06:14:31,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/ba051e2918f746f0877a827d9afd154b is 50, key is test_row_0/C:col10/1733811270956/Put/seqid=0 2024-12-10T06:14:31,472 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/278ff28185bb47c59a87649fd9631c1b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/278ff28185bb47c59a87649fd9631c1b 2024-12-10T06:14:31,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741919_1095 (size=12301) 2024-12-10T06:14:31,481 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/B of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 278ff28185bb47c59a87649fd9631c1b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:31,481 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:31,481 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/B, priority=13, startTime=1733811270968; duration=0sec 2024-12-10T06:14:31,481 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:31,481 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:B 2024-12-10T06:14:31,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/ba051e2918f746f0877a827d9afd154b 2024-12-10T06:14:31,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/cc4977f36c1f48cfb9a04f19c55f2fb8 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/cc4977f36c1f48cfb9a04f19c55f2fb8 2024-12-10T06:14:31,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/cc4977f36c1f48cfb9a04f19c55f2fb8, entries=200, sequenceid=391, filesize=14.4 K 2024-12-10T06:14:31,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/eb54e0924dce4c5bb0443d00c27188e1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/eb54e0924dce4c5bb0443d00c27188e1 2024-12-10T06:14:31,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/eb54e0924dce4c5bb0443d00c27188e1, entries=150, sequenceid=391, filesize=12.0 K 2024-12-10T06:14:31,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/ba051e2918f746f0877a827d9afd154b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/ba051e2918f746f0877a827d9afd154b 2024-12-10T06:14:31,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/ba051e2918f746f0877a827d9afd154b, entries=150, sequenceid=391, filesize=12.0 K 2024-12-10T06:14:31,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 544ms, sequenceid=391, compaction requested=true 2024-12-10T06:14:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:31,513 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T06:14:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:31,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:31,514 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T06:14:31,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:31,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:31,514 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T06:14:31,514 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T06:14:31,514 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. because compaction request was cancelled 2024-12-10T06:14:31,515 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:A 2024-12-10T06:14:31,515 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:31,515 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T06:14:31,515 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T06:14:31,515 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. because compaction request was cancelled 2024-12-10T06:14:31,515 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:B 2024-12-10T06:14:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T06:14:31,516 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:31,516 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/C is initiating minor compaction (all files) 2024-12-10T06:14:31,517 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/C in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:31,517 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0c5ddd38726646e18c67f89cd6ab66af, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/095d8aa3c371493f9cc93e5271ed977f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/727a22c84a064df39936dfaf6669f8f1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/ba051e2918f746f0877a827d9afd154b] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=48.8 K 2024-12-10T06:14:31,517 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c5ddd38726646e18c67f89cd6ab66af, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733811269520 2024-12-10T06:14:31,518 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 095d8aa3c371493f9cc93e5271ed977f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733811270130 2024-12-10T06:14:31,518 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 727a22c84a064df39936dfaf6669f8f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733811270718 2024-12-10T06:14:31,519 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba051e2918f746f0877a827d9afd154b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733811270924 2024-12-10T06:14:31,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T06:14:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:31,521 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:14:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:31,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/34bc8ef85e684b9d8ba9f9f586005446 is 50, key is test_row_0/A:col10/1733811271068/Put/seqid=0 2024-12-10T06:14:31,552 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#C#compaction#84 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:31,553 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/4c44b704d79445c2af506976422ca8f9 is 50, key is test_row_0/C:col10/1733811270956/Put/seqid=0 2024-12-10T06:14:31,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741920_1096 (size=12301) 2024-12-10T06:14:31,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/34bc8ef85e684b9d8ba9f9f586005446 2024-12-10T06:14:31,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/ebd34e2b411944b6a9b2293ba78ab8dd is 50, key is test_row_0/B:col10/1733811271068/Put/seqid=0 2024-12-10T06:14:31,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741921_1097 (size=13221) 2024-12-10T06:14:31,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741922_1098 (size=12301) 2024-12-10T06:14:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:31,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:31,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811331814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:31,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:31,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811331926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:32,009 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/4c44b704d79445c2af506976422ca8f9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/4c44b704d79445c2af506976422ca8f9 2024-12-10T06:14:32,019 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/C of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 4c44b704d79445c2af506976422ca8f9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:32,019 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:32,019 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/C, priority=12, startTime=1733811271514; duration=0sec 2024-12-10T06:14:32,020 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:32,020 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:C 2024-12-10T06:14:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T06:14:32,035 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/ebd34e2b411944b6a9b2293ba78ab8dd 2024-12-10T06:14:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/2afec54da0a94a3ca187e3345cc124ea is 50, key is test_row_0/C:col10/1733811271068/Put/seqid=0 2024-12-10T06:14:32,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741923_1099 (size=12301) 2024-12-10T06:14:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811332135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50302 deadline: 1733811332305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:32,307 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 91cf7a8c24ad1cfeb344be5ce72aa8a5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:32,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50316 deadline: 1733811332315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:32,320 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 91cf7a8c24ad1cfeb344be5ce72aa8a5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:32,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811332445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:32,460 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/2afec54da0a94a3ca187e3345cc124ea 2024-12-10T06:14:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/34bc8ef85e684b9d8ba9f9f586005446 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/34bc8ef85e684b9d8ba9f9f586005446 2024-12-10T06:14:32,475 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/34bc8ef85e684b9d8ba9f9f586005446, entries=150, sequenceid=410, filesize=12.0 K 2024-12-10T06:14:32,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/ebd34e2b411944b6a9b2293ba78ab8dd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/ebd34e2b411944b6a9b2293ba78ab8dd 2024-12-10T06:14:32,483 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/ebd34e2b411944b6a9b2293ba78ab8dd, entries=150, sequenceid=410, filesize=12.0 K 2024-12-10T06:14:32,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/2afec54da0a94a3ca187e3345cc124ea as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2afec54da0a94a3ca187e3345cc124ea 2024-12-10T06:14:32,490 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2afec54da0a94a3ca187e3345cc124ea, entries=150, sequenceid=410, filesize=12.0 K 2024-12-10T06:14:32,492 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 971ms, sequenceid=410, compaction requested=true 2024-12-10T06:14:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-10T06:14:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-10T06:14:32,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-10T06:14:32,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6050 sec 2024-12-10T06:14:32,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6140 sec 2024-12-10T06:14:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:32,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T06:14:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:32,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/f5e3a4136861431f9a6a75c954753d34 is 50, key is test_row_0/A:col10/1733811271775/Put/seqid=0 2024-12-10T06:14:33,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741924_1100 (size=14741) 2024-12-10T06:14:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T06:14:33,035 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:33,036 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-10T06:14:33,039 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T06:14:33,041 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:33,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:33,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811333052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T06:14:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811333165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:33,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T06:14:33,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:33,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T06:14:33,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T06:14:33,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811333375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:33,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:33,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:33,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:33,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/f5e3a4136861431f9a6a75c954753d34 2024-12-10T06:14:33,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/e9a98b7e8a834641ac38d5970852f3f4 is 50, key is test_row_0/B:col10/1733811271775/Put/seqid=0 2024-12-10T06:14:33,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741925_1101 (size=12301) 2024-12-10T06:14:33,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T06:14:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T06:14:33,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T06:14:33,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50326 deadline: 1733811333685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:33,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:33,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:33,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:14:33,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T06:14:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:33,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/e9a98b7e8a834641ac38d5970852f3f4 2024-12-10T06:14:33,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/e780944ece3f401190ed5d02b5029ad6 is 50, key is test_row_0/C:col10/1733811271775/Put/seqid=0 2024-12-10T06:14:33,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741926_1102 (size=12301) 2024-12-10T06:14:33,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/e780944ece3f401190ed5d02b5029ad6 2024-12-10T06:14:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/f5e3a4136861431f9a6a75c954753d34 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5e3a4136861431f9a6a75c954753d34 2024-12-10T06:14:33,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5e3a4136861431f9a6a75c954753d34, entries=200, sequenceid=428, filesize=14.4 K 2024-12-10T06:14:33,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/e9a98b7e8a834641ac38d5970852f3f4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/e9a98b7e8a834641ac38d5970852f3f4 2024-12-10T06:14:33,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/e9a98b7e8a834641ac38d5970852f3f4, entries=150, sequenceid=428, filesize=12.0 K 2024-12-10T06:14:33,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/e780944ece3f401190ed5d02b5029ad6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e780944ece3f401190ed5d02b5029ad6 2024-12-10T06:14:33,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e780944ece3f401190ed5d02b5029ad6, entries=150, sequenceid=428, filesize=12.0 K 2024-12-10T06:14:33,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 906ms, sequenceid=428, compaction requested=true 2024-12-10T06:14:33,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:33,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:33,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:33,866 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:33,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:33,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:33,866 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:33,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91cf7a8c24ad1cfeb344be5ce72aa8a5:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:33,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:33,868 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:33,868 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/A is initiating minor compaction (all files) 2024-12-10T06:14:33,868 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/A in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,868 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9e1e2b309d7649a09c6810aee81bbcad, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/cc4977f36c1f48cfb9a04f19c55f2fb8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/34bc8ef85e684b9d8ba9f9f586005446, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5e3a4136861431f9a6a75c954753d34] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=53.7 K 2024-12-10T06:14:33,868 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:33,869 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/B is initiating minor compaction (all files) 2024-12-10T06:14:33,869 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/B in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,869 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/278ff28185bb47c59a87649fd9631c1b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/eb54e0924dce4c5bb0443d00c27188e1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/ebd34e2b411944b6a9b2293ba78ab8dd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/e9a98b7e8a834641ac38d5970852f3f4] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=48.9 K 2024-12-10T06:14:33,872 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e1e2b309d7649a09c6810aee81bbcad, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733811270718 2024-12-10T06:14:33,872 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 278ff28185bb47c59a87649fd9631c1b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733811270718 2024-12-10T06:14:33,873 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting eb54e0924dce4c5bb0443d00c27188e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733811270924 2024-12-10T06:14:33,873 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc4977f36c1f48cfb9a04f19c55f2fb8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733811270924 2024-12-10T06:14:33,874 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34bc8ef85e684b9d8ba9f9f586005446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733811271044 2024-12-10T06:14:33,874 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd34e2b411944b6a9b2293ba78ab8dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733811271044 2024-12-10T06:14:33,874 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e3a4136861431f9a6a75c954753d34, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733811271775 2024-12-10T06:14:33,874 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a98b7e8a834641ac38d5970852f3f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733811271775 2024-12-10T06:14:33,888 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:33,889 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/e2a666f59f974a41846de586321ac860 is 50, key is test_row_0/A:col10/1733811271775/Put/seqid=0 2024-12-10T06:14:33,891 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:33,892 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/267893c0263a4128a8f99e0e094164dd is 50, key is test_row_0/B:col10/1733811271775/Put/seqid=0 2024-12-10T06:14:33,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741927_1103 (size=13323) 2024-12-10T06:14:33,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741928_1104 (size=13323) 2024-12-10T06:14:33,913 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-10T06:14:33,913 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-10T06:14:33,914 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:14:33,914 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:14:33,917 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-10T06:14:33,917 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:14:33,919 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:33,919 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:33,919 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:33,920 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:33,920 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:33,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:33,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:33,920 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:33,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:33,921 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-10T06:14:33,921 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:14:33,921 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:33,921 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:33,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:33,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T06:14:33,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:33,966 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T06:14:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/95d8689f1671442aa670fce5eb018d38 is 50, key is test_row_0/A:col10/1733811273045/Put/seqid=0 2024-12-10T06:14:33,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741929_1105 (size=12301) 2024-12-10T06:14:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T06:14:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:34,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. as already flushing 2024-12-10T06:14:34,198 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-10T06:14:34,198 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:14:34,198 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:34,198 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:34,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:34,304 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/e2a666f59f974a41846de586321ac860 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/e2a666f59f974a41846de586321ac860 2024-12-10T06:14:34,305 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/267893c0263a4128a8f99e0e094164dd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/267893c0263a4128a8f99e0e094164dd 2024-12-10T06:14:34,311 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/B of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 267893c0263a4128a8f99e0e094164dd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:34,311 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:34,311 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/B, priority=12, startTime=1733811273866; duration=0sec 2024-12-10T06:14:34,311 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:34,311 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:B 2024-12-10T06:14:34,311 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/A of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into e2a666f59f974a41846de586321ac860(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:34,311 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:34,311 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:34,311 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/A, priority=12, startTime=1733811273866; duration=0sec 2024-12-10T06:14:34,311 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:34,311 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:A 2024-12-10T06:14:34,312 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:34,312 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 91cf7a8c24ad1cfeb344be5ce72aa8a5/C is initiating minor compaction (all files) 2024-12-10T06:14:34,312 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91cf7a8c24ad1cfeb344be5ce72aa8a5/C in TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:34,312 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/4c44b704d79445c2af506976422ca8f9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2afec54da0a94a3ca187e3345cc124ea, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e780944ece3f401190ed5d02b5029ad6] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp, totalSize=36.9 K 2024-12-10T06:14:34,313 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c44b704d79445c2af506976422ca8f9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733811270924 2024-12-10T06:14:34,313 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2afec54da0a94a3ca187e3345cc124ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733811271044 2024-12-10T06:14:34,313 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting e780944ece3f401190ed5d02b5029ad6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733811271775 2024-12-10T06:14:34,320 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91cf7a8c24ad1cfeb344be5ce72aa8a5#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:34,321 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/29ca1cdd7cbc41ed93b447a4f38042be is 50, key is test_row_0/C:col10/1733811271775/Put/seqid=0 2024-12-10T06:14:34,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741930_1106 (size=13323) 2024-12-10T06:14:34,337 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-10T06:14:34,337 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:14:34,337 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:34,337 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:34,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:34,346 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-10T06:14:34,346 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:14:34,346 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:34,346 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:34,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:34,376 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/95d8689f1671442aa670fce5eb018d38 2024-12-10T06:14:34,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3ea4a173135443cc9a44b1a26a7d842a is 50, key is test_row_0/B:col10/1733811273045/Put/seqid=0 2024-12-10T06:14:34,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741931_1107 (size=12301) 2024-12-10T06:14:34,731 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/29ca1cdd7cbc41ed93b447a4f38042be as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/29ca1cdd7cbc41ed93b447a4f38042be 2024-12-10T06:14:34,736 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91cf7a8c24ad1cfeb344be5ce72aa8a5/C of 91cf7a8c24ad1cfeb344be5ce72aa8a5 into 29ca1cdd7cbc41ed93b447a4f38042be(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:34,736 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:34,736 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., storeName=91cf7a8c24ad1cfeb344be5ce72aa8a5/C, priority=13, startTime=1733811273866; duration=0sec 2024-12-10T06:14:34,736 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:34,736 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91cf7a8c24ad1cfeb344be5ce72aa8a5:C 2024-12-10T06:14:34,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3ea4a173135443cc9a44b1a26a7d842a 2024-12-10T06:14:34,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/e2019d0d660441b3b2056ad634c85862 is 50, key is test_row_0/C:col10/1733811273045/Put/seqid=0 2024-12-10T06:14:34,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741932_1108 (size=12301) 2024-12-10T06:14:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T06:14:35,208 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/e2019d0d660441b3b2056ad634c85862 2024-12-10T06:14:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/95d8689f1671442aa670fce5eb018d38 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/95d8689f1671442aa670fce5eb018d38 2024-12-10T06:14:35,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/95d8689f1671442aa670fce5eb018d38, entries=150, sequenceid=443, filesize=12.0 K 2024-12-10T06:14:35,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3ea4a173135443cc9a44b1a26a7d842a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3ea4a173135443cc9a44b1a26a7d842a 2024-12-10T06:14:35,230 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3ea4a173135443cc9a44b1a26a7d842a, entries=150, sequenceid=443, filesize=12.0 K 2024-12-10T06:14:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/e2019d0d660441b3b2056ad634c85862 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e2019d0d660441b3b2056ad634c85862 2024-12-10T06:14:35,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e2019d0d660441b3b2056ad634c85862, entries=150, sequenceid=443, filesize=12.0 K 2024-12-10T06:14:35,239 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 1274ms, sequenceid=443, compaction requested=false 2024-12-10T06:14:35,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: 2024-12-10T06:14:35,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:35,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-10T06:14:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-10T06:14:35,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-10T06:14:35,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2010 sec 2024-12-10T06:14:35,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.2090 sec 2024-12-10T06:14:35,915 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T06:14:36,658 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-10T06:14:36,658 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:14:36,658 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:36,658 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:36,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:36,667 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-10T06:14:36,667 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:14:36,668 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:36,668 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:36,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T06:14:37,206 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2413 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2430 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T06:14:37,206 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1092 2024-12-10T06:14:37,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3268 rows 2024-12-10T06:14:37,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1090 2024-12-10T06:14:37,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3269 rows 2024-12-10T06:14:37,207 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:14:37,207 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:14:37,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:37,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:37,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:14:37,211 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T06:14:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T06:14:37,220 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811277220"}]},"ts":"1733811277220"} 2024-12-10T06:14:37,223 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T06:14:37,223 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T06:14:37,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T06:14:37,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91cf7a8c24ad1cfeb344be5ce72aa8a5, UNASSIGN}] 2024-12-10T06:14:37,230 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91cf7a8c24ad1cfeb344be5ce72aa8a5, UNASSIGN 2024-12-10T06:14:37,231 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=91cf7a8c24ad1cfeb344be5ce72aa8a5, regionState=CLOSING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:37,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91cf7a8c24ad1cfeb344be5ce72aa8a5, UNASSIGN because future has completed 2024-12-10T06:14:37,234 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:14:37,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:14:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T06:14:37,391 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:37,391 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:14:37,392 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 91cf7a8c24ad1cfeb344be5ce72aa8a5, disabling compactions & flushes 2024-12-10T06:14:37,392 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:37,392 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:37,392 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. after waiting 0 ms 2024-12-10T06:14:37,392 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:37,392 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T06:14:37,393 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=A 2024-12-10T06:14:37,393 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:37,393 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=B 2024-12-10T06:14:37,393 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:37,393 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91cf7a8c24ad1cfeb344be5ce72aa8a5, store=C 2024-12-10T06:14:37,393 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:37,397 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/f5a5d4af04a34ee7a1a73870dfd66185 is 50, key is test_row_0/A:col10/1733811274196/Put/seqid=0 2024-12-10T06:14:37,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741933_1109 (size=12301) 2024-12-10T06:14:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T06:14:37,802 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/f5a5d4af04a34ee7a1a73870dfd66185 2024-12-10T06:14:37,810 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3f2cfc5e508448daaa28fa64ab861b22 is 50, key is test_row_0/B:col10/1733811274196/Put/seqid=0 2024-12-10T06:14:37,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741934_1110 (size=12301) 2024-12-10T06:14:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T06:14:38,214 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3f2cfc5e508448daaa28fa64ab861b22 2024-12-10T06:14:38,223 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/68337b5a459a41df9b259eb4862e17fe is 50, key is test_row_0/C:col10/1733811274196/Put/seqid=0 2024-12-10T06:14:38,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741935_1111 (size=12301) 2024-12-10T06:14:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T06:14:38,628 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/68337b5a459a41df9b259eb4862e17fe 2024-12-10T06:14:38,633 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/A/f5a5d4af04a34ee7a1a73870dfd66185 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5a5d4af04a34ee7a1a73870dfd66185 2024-12-10T06:14:38,637 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5a5d4af04a34ee7a1a73870dfd66185, entries=150, sequenceid=454, filesize=12.0 K 2024-12-10T06:14:38,638 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/B/3f2cfc5e508448daaa28fa64ab861b22 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3f2cfc5e508448daaa28fa64ab861b22 2024-12-10T06:14:38,642 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3f2cfc5e508448daaa28fa64ab861b22, entries=150, sequenceid=454, filesize=12.0 K 2024-12-10T06:14:38,643 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/.tmp/C/68337b5a459a41df9b259eb4862e17fe as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/68337b5a459a41df9b259eb4862e17fe 2024-12-10T06:14:38,647 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/68337b5a459a41df9b259eb4862e17fe, entries=150, sequenceid=454, filesize=12.0 K 2024-12-10T06:14:38,648 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 1256ms, sequenceid=454, compaction requested=true 2024-12-10T06:14:38,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/713ea3f7a5144e31923a21a444b317e9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/577e72e833f143f9b5744ecbc8e31640, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/2a924a6225db4f3baba79f9275723211, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/311a874dcfc546029655781058751ab8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4aa4a313063a4c33a3241060121ff3f2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/49a8fd7e9bfb40508356757cd0420396, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dc79dd94682746a9bfaecf606ed19448, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/121bf73ad88347d29bcd48e95e127a45, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d8f8cc135aec4b0e9083d685d5c59b67, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/e757c74ffad54262b80a6d4012b8d9f9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dfcbcde06bd841ca93d456053803a2b6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d52eec660ce24f7e9d0d486c3a1883f6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a0f828aa1f2a4278916fbd1757ebf824, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f35de441a3124d1e838757ed28f01085, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9f9f04d718b84d5bad84a68a630bd96d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c10ea3432c5b4933b036045c076ba011, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/47fedb90c0c94d1ea35e52bdc812a968, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/3eaa5ac0d4bc42b0bbeb6e64febf192c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/6c25c5f8f9e84f8aba19a2506b0ae738, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a85ab9b97d0e48528a68bf88a36cab32, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/090e5b35f74b47ad99759b153b256f7e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c9ef47dbbebc454eb30d8a5eb3cdd417, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4747112d5fdd4546b9ef6b4e43bdbd64, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/ec4544ee74094c6fa93502dc6c28dfeb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/8e9324d8491445198b5bed0ac295315c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c8805fb7f3ed4cb2bc5319d1a4e3b5ba, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9e1e2b309d7649a09c6810aee81bbcad, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/cc4977f36c1f48cfb9a04f19c55f2fb8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/34bc8ef85e684b9d8ba9f9f586005446, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5e3a4136861431f9a6a75c954753d34] to archive 2024-12-10T06:14:38,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:14:38,659 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/577e72e833f143f9b5744ecbc8e31640 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/577e72e833f143f9b5744ecbc8e31640 2024-12-10T06:14:38,659 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4aa4a313063a4c33a3241060121ff3f2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4aa4a313063a4c33a3241060121ff3f2 2024-12-10T06:14:38,659 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/49a8fd7e9bfb40508356757cd0420396 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/49a8fd7e9bfb40508356757cd0420396 2024-12-10T06:14:38,659 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/121bf73ad88347d29bcd48e95e127a45 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/121bf73ad88347d29bcd48e95e127a45 2024-12-10T06:14:38,660 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/311a874dcfc546029655781058751ab8 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/311a874dcfc546029655781058751ab8 2024-12-10T06:14:38,660 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/2a924a6225db4f3baba79f9275723211 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/2a924a6225db4f3baba79f9275723211 2024-12-10T06:14:38,660 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dc79dd94682746a9bfaecf606ed19448 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dc79dd94682746a9bfaecf606ed19448 2024-12-10T06:14:38,661 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/713ea3f7a5144e31923a21a444b317e9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/713ea3f7a5144e31923a21a444b317e9 2024-12-10T06:14:38,661 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d8f8cc135aec4b0e9083d685d5c59b67 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d8f8cc135aec4b0e9083d685d5c59b67 2024-12-10T06:14:38,662 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a0f828aa1f2a4278916fbd1757ebf824 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a0f828aa1f2a4278916fbd1757ebf824 2024-12-10T06:14:38,662 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dfcbcde06bd841ca93d456053803a2b6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/dfcbcde06bd841ca93d456053803a2b6 2024-12-10T06:14:38,662 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/e757c74ffad54262b80a6d4012b8d9f9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/e757c74ffad54262b80a6d4012b8d9f9 2024-12-10T06:14:38,663 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9f9f04d718b84d5bad84a68a630bd96d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9f9f04d718b84d5bad84a68a630bd96d 2024-12-10T06:14:38,663 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d52eec660ce24f7e9d0d486c3a1883f6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/d52eec660ce24f7e9d0d486c3a1883f6 2024-12-10T06:14:38,664 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c10ea3432c5b4933b036045c076ba011 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c10ea3432c5b4933b036045c076ba011 2024-12-10T06:14:38,664 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f35de441a3124d1e838757ed28f01085 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f35de441a3124d1e838757ed28f01085 2024-12-10T06:14:38,664 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/47fedb90c0c94d1ea35e52bdc812a968 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/47fedb90c0c94d1ea35e52bdc812a968 2024-12-10T06:14:38,665 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a85ab9b97d0e48528a68bf88a36cab32 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/a85ab9b97d0e48528a68bf88a36cab32 2024-12-10T06:14:38,665 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/6c25c5f8f9e84f8aba19a2506b0ae738 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/6c25c5f8f9e84f8aba19a2506b0ae738 2024-12-10T06:14:38,665 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/090e5b35f74b47ad99759b153b256f7e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/090e5b35f74b47ad99759b153b256f7e 2024-12-10T06:14:38,666 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/3eaa5ac0d4bc42b0bbeb6e64febf192c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/3eaa5ac0d4bc42b0bbeb6e64febf192c 2024-12-10T06:14:38,666 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c9ef47dbbebc454eb30d8a5eb3cdd417 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c9ef47dbbebc454eb30d8a5eb3cdd417 2024-12-10T06:14:38,666 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/ec4544ee74094c6fa93502dc6c28dfeb to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/ec4544ee74094c6fa93502dc6c28dfeb 2024-12-10T06:14:38,667 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4747112d5fdd4546b9ef6b4e43bdbd64 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/4747112d5fdd4546b9ef6b4e43bdbd64 2024-12-10T06:14:38,667 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/8e9324d8491445198b5bed0ac295315c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/8e9324d8491445198b5bed0ac295315c 2024-12-10T06:14:38,667 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c8805fb7f3ed4cb2bc5319d1a4e3b5ba to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/c8805fb7f3ed4cb2bc5319d1a4e3b5ba 2024-12-10T06:14:38,668 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/cc4977f36c1f48cfb9a04f19c55f2fb8 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/cc4977f36c1f48cfb9a04f19c55f2fb8 2024-12-10T06:14:38,668 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/34bc8ef85e684b9d8ba9f9f586005446 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/34bc8ef85e684b9d8ba9f9f586005446 2024-12-10T06:14:38,668 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5e3a4136861431f9a6a75c954753d34 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5e3a4136861431f9a6a75c954753d34 2024-12-10T06:14:38,668 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9e1e2b309d7649a09c6810aee81bbcad to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/9e1e2b309d7649a09c6810aee81bbcad 2024-12-10T06:14:38,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8c045b698b1241ed9c21c5208208281c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/d651a49ee8ab411a9b6df157507e7b74, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3418c204d92b411597a890b867b3f9a7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8046ae8979ff479eb5d1fb38d3b6dae0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/c8d68aefc43b4e7bac8efd41fa0dcc9d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/66545074372f4754839dc673cf33c911, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3db7f71733814f2297adbb37fce68e02, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/6f2a36cbdeb34ba688b4dfa0d82060af, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/2de0a10630514d3fbaa147beecd1ecda, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/cc93bb07204b4be48be5cb6df204d33e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/35b59f611e324e8ea8e8315d5c130dd5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b0db0263c1c64ea8ac19286f80df854b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/740dcc45f6c944f7b3cf11535962e82a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/239a33ba65a648148f5f24687cb9a42f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9fc53ee2289e4d18a25c2edb4f65e64b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/f458f54d53f640298effde81fc60e844, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/92f06fefaa3f469c8baaaa6e357cc4d5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/5b65b55e56ad4566b0687ca6e8016034, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b4e0d841fbd8471f931e06cf8f88eca0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9ff01d0a46304b419eb1bc67c8c966d9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/63a67bc4a35f425a98476249e0e5cda6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8ae8125cda3a4bb4880cfc5ef09cc897, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8719aba1a6eb4bbf85aedc89f57d325c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/395aa80536ec478284677c7aa75b3876, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/0e335dcbd24342bb9dfb5fe02ad078a5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/278ff28185bb47c59a87649fd9631c1b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/fb9c26a0e8914274a2cd2f01e7d3eabe, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/eb54e0924dce4c5bb0443d00c27188e1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/ebd34e2b411944b6a9b2293ba78ab8dd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/e9a98b7e8a834641ac38d5970852f3f4] to archive 2024-12-10T06:14:38,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:14:38,686 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3418c204d92b411597a890b867b3f9a7 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3418c204d92b411597a890b867b3f9a7 2024-12-10T06:14:38,686 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/c8d68aefc43b4e7bac8efd41fa0dcc9d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/c8d68aefc43b4e7bac8efd41fa0dcc9d 2024-12-10T06:14:38,686 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/66545074372f4754839dc673cf33c911 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/66545074372f4754839dc673cf33c911 2024-12-10T06:14:38,686 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/d651a49ee8ab411a9b6df157507e7b74 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/d651a49ee8ab411a9b6df157507e7b74 2024-12-10T06:14:38,686 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8c045b698b1241ed9c21c5208208281c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8c045b698b1241ed9c21c5208208281c 2024-12-10T06:14:38,687 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3db7f71733814f2297adbb37fce68e02 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3db7f71733814f2297adbb37fce68e02 2024-12-10T06:14:38,687 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/6f2a36cbdeb34ba688b4dfa0d82060af to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/6f2a36cbdeb34ba688b4dfa0d82060af 2024-12-10T06:14:38,687 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8046ae8979ff479eb5d1fb38d3b6dae0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8046ae8979ff479eb5d1fb38d3b6dae0 2024-12-10T06:14:38,689 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/2de0a10630514d3fbaa147beecd1ecda to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/2de0a10630514d3fbaa147beecd1ecda 2024-12-10T06:14:38,689 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/cc93bb07204b4be48be5cb6df204d33e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/cc93bb07204b4be48be5cb6df204d33e 2024-12-10T06:14:38,689 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/35b59f611e324e8ea8e8315d5c130dd5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/35b59f611e324e8ea8e8315d5c130dd5 2024-12-10T06:14:38,690 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b0db0263c1c64ea8ac19286f80df854b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b0db0263c1c64ea8ac19286f80df854b 2024-12-10T06:14:38,690 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9fc53ee2289e4d18a25c2edb4f65e64b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9fc53ee2289e4d18a25c2edb4f65e64b 2024-12-10T06:14:38,690 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/239a33ba65a648148f5f24687cb9a42f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/239a33ba65a648148f5f24687cb9a42f 2024-12-10T06:14:38,690 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/f458f54d53f640298effde81fc60e844 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/f458f54d53f640298effde81fc60e844 2024-12-10T06:14:38,691 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/92f06fefaa3f469c8baaaa6e357cc4d5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/92f06fefaa3f469c8baaaa6e357cc4d5 2024-12-10T06:14:38,692 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b4e0d841fbd8471f931e06cf8f88eca0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/b4e0d841fbd8471f931e06cf8f88eca0 2024-12-10T06:14:38,692 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/5b65b55e56ad4566b0687ca6e8016034 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/5b65b55e56ad4566b0687ca6e8016034 2024-12-10T06:14:38,692 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/63a67bc4a35f425a98476249e0e5cda6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/63a67bc4a35f425a98476249e0e5cda6 2024-12-10T06:14:38,692 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9ff01d0a46304b419eb1bc67c8c966d9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/9ff01d0a46304b419eb1bc67c8c966d9 2024-12-10T06:14:38,692 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/740dcc45f6c944f7b3cf11535962e82a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/740dcc45f6c944f7b3cf11535962e82a 2024-12-10T06:14:38,692 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8ae8125cda3a4bb4880cfc5ef09cc897 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8ae8125cda3a4bb4880cfc5ef09cc897 2024-12-10T06:14:38,693 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8719aba1a6eb4bbf85aedc89f57d325c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/8719aba1a6eb4bbf85aedc89f57d325c 2024-12-10T06:14:38,694 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/278ff28185bb47c59a87649fd9631c1b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/278ff28185bb47c59a87649fd9631c1b 2024-12-10T06:14:38,694 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/eb54e0924dce4c5bb0443d00c27188e1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/eb54e0924dce4c5bb0443d00c27188e1 2024-12-10T06:14:38,694 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/395aa80536ec478284677c7aa75b3876 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/395aa80536ec478284677c7aa75b3876 2024-12-10T06:14:38,694 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/0e335dcbd24342bb9dfb5fe02ad078a5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/0e335dcbd24342bb9dfb5fe02ad078a5 2024-12-10T06:14:38,695 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/fb9c26a0e8914274a2cd2f01e7d3eabe to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/fb9c26a0e8914274a2cd2f01e7d3eabe 2024-12-10T06:14:38,695 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/e9a98b7e8a834641ac38d5970852f3f4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/e9a98b7e8a834641ac38d5970852f3f4 2024-12-10T06:14:38,695 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/ebd34e2b411944b6a9b2293ba78ab8dd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/ebd34e2b411944b6a9b2293ba78ab8dd 2024-12-10T06:14:38,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0ed9ec52dc8448eaa36ed0e965df1fc0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/dd693241781d46fdaa1bb40b4067d59e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8b7cbbd7d942442585103ba5a2bd9f29, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/df47a1313b624d44bfe24b81317252d4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/bc54847c7ccd4df786645b734df35e09, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/9ff7109bb51c4056a39479b860b14036, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2bc5ac6f4b6e4d518143f20b29dd006a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8bf7063a0d1e4ac8ad3bf5b93244c00f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/79a50f782fa74c85a26a8555fbee5669, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0d850dc92d6f45a8877f0ba17477da0a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/7ecfa963fbb1438e83ec2121c3f54789, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8333da3fac0148dd887c1972e1417b28, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/03f7b589ccec49628f15849f74702409, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/730b1b34ced8426e8c67802bed00bf63, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/87fb15ae0d194bfe92f0928c7b5f04cd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/725c892df10e47ce9c26bc4c93b525dd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/da2e590ef1b94723af1466bab839080e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/40299e4f32f143008ddb98c5a9baf303, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/b003496f69e34f21ba4a1eec9263591d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/6404f88eb2044287913651a9fd281778, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/c643d712a77740f9a64b6ba1255b7dc1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/d7950b91a13b4d2695ed57727e6749a6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0c5ddd38726646e18c67f89cd6ab66af, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0151f8b4dd1345129c9bb780c1036b79, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/095d8aa3c371493f9cc93e5271ed977f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/727a22c84a064df39936dfaf6669f8f1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/4c44b704d79445c2af506976422ca8f9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/ba051e2918f746f0877a827d9afd154b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2afec54da0a94a3ca187e3345cc124ea, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e780944ece3f401190ed5d02b5029ad6] to archive 2024-12-10T06:14:38,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:14:38,700 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/df47a1313b624d44bfe24b81317252d4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/df47a1313b624d44bfe24b81317252d4 2024-12-10T06:14:38,700 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0ed9ec52dc8448eaa36ed0e965df1fc0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0ed9ec52dc8448eaa36ed0e965df1fc0 2024-12-10T06:14:38,700 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/dd693241781d46fdaa1bb40b4067d59e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/dd693241781d46fdaa1bb40b4067d59e 2024-12-10T06:14:38,700 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2bc5ac6f4b6e4d518143f20b29dd006a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2bc5ac6f4b6e4d518143f20b29dd006a 2024-12-10T06:14:38,700 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/bc54847c7ccd4df786645b734df35e09 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/bc54847c7ccd4df786645b734df35e09 2024-12-10T06:14:38,701 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8bf7063a0d1e4ac8ad3bf5b93244c00f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8bf7063a0d1e4ac8ad3bf5b93244c00f 2024-12-10T06:14:38,701 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/9ff7109bb51c4056a39479b860b14036 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/9ff7109bb51c4056a39479b860b14036 2024-12-10T06:14:38,701 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8b7cbbd7d942442585103ba5a2bd9f29 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8b7cbbd7d942442585103ba5a2bd9f29 2024-12-10T06:14:38,702 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0d850dc92d6f45a8877f0ba17477da0a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0d850dc92d6f45a8877f0ba17477da0a 2024-12-10T06:14:38,702 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/7ecfa963fbb1438e83ec2121c3f54789 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/7ecfa963fbb1438e83ec2121c3f54789 2024-12-10T06:14:38,703 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/79a50f782fa74c85a26a8555fbee5669 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/79a50f782fa74c85a26a8555fbee5669 2024-12-10T06:14:38,703 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8333da3fac0148dd887c1972e1417b28 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/8333da3fac0148dd887c1972e1417b28 2024-12-10T06:14:38,703 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/03f7b589ccec49628f15849f74702409 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/03f7b589ccec49628f15849f74702409 2024-12-10T06:14:38,703 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/87fb15ae0d194bfe92f0928c7b5f04cd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/87fb15ae0d194bfe92f0928c7b5f04cd 2024-12-10T06:14:38,703 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/725c892df10e47ce9c26bc4c93b525dd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/725c892df10e47ce9c26bc4c93b525dd 2024-12-10T06:14:38,704 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/730b1b34ced8426e8c67802bed00bf63 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/730b1b34ced8426e8c67802bed00bf63 2024-12-10T06:14:38,705 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/40299e4f32f143008ddb98c5a9baf303 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/40299e4f32f143008ddb98c5a9baf303 2024-12-10T06:14:38,705 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/6404f88eb2044287913651a9fd281778 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/6404f88eb2044287913651a9fd281778 2024-12-10T06:14:38,705 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/c643d712a77740f9a64b6ba1255b7dc1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/c643d712a77740f9a64b6ba1255b7dc1 2024-12-10T06:14:38,705 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/b003496f69e34f21ba4a1eec9263591d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/b003496f69e34f21ba4a1eec9263591d 2024-12-10T06:14:38,706 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/da2e590ef1b94723af1466bab839080e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/da2e590ef1b94723af1466bab839080e 2024-12-10T06:14:38,706 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/d7950b91a13b4d2695ed57727e6749a6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/d7950b91a13b4d2695ed57727e6749a6 2024-12-10T06:14:38,706 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0c5ddd38726646e18c67f89cd6ab66af to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0c5ddd38726646e18c67f89cd6ab66af 2024-12-10T06:14:38,707 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0151f8b4dd1345129c9bb780c1036b79 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/0151f8b4dd1345129c9bb780c1036b79 2024-12-10T06:14:38,707 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/095d8aa3c371493f9cc93e5271ed977f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/095d8aa3c371493f9cc93e5271ed977f 2024-12-10T06:14:38,707 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/727a22c84a064df39936dfaf6669f8f1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/727a22c84a064df39936dfaf6669f8f1 2024-12-10T06:14:38,708 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/ba051e2918f746f0877a827d9afd154b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/ba051e2918f746f0877a827d9afd154b 2024-12-10T06:14:38,708 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/4c44b704d79445c2af506976422ca8f9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/4c44b704d79445c2af506976422ca8f9 2024-12-10T06:14:38,708 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2afec54da0a94a3ca187e3345cc124ea to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/2afec54da0a94a3ca187e3345cc124ea 2024-12-10T06:14:38,709 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e780944ece3f401190ed5d02b5029ad6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e780944ece3f401190ed5d02b5029ad6 2024-12-10T06:14:38,714 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/recovered.edits/457.seqid, newMaxSeqId=457, maxSeqId=1 2024-12-10T06:14:38,715 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. 2024-12-10T06:14:38,715 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 91cf7a8c24ad1cfeb344be5ce72aa8a5: Waiting for close lock at 1733811277392Running coprocessor pre-close hooks at 1733811277392Disabling compacts and flushes for region at 1733811277392Disabling writes for close at 1733811277392Obtaining lock to block concurrent updates at 1733811277392Preparing flush snapshotting stores in 91cf7a8c24ad1cfeb344be5ce72aa8a5 at 1733811277392Finished memstore snapshotting TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733811277393 (+1 ms)Flushing stores of TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5. at 1733811277393Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5/A: creating writer at 1733811277394 (+1 ms)Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5/A: appending metadata at 1733811277397 (+3 ms)Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5/A: closing flushed file at 1733811277397Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5/B: creating writer at 1733811277806 (+409 ms)Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5/B: appending metadata at 1733811277809 (+3 ms)Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5/B: closing flushed file at 1733811277809Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5/C: creating writer at 1733811278219 (+410 ms)Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5/C: appending metadata at 1733811278222 (+3 ms)Flushing 91cf7a8c24ad1cfeb344be5ce72aa8a5/C: closing flushed file at 1733811278223 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e7ce3fd: reopening flushed file at 1733811278633 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e93e24: reopening flushed file at 1733811278638 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fa585e8: reopening flushed file at 1733811278643 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 91cf7a8c24ad1cfeb344be5ce72aa8a5 in 1256ms, sequenceid=454, compaction requested=true at 1733811278648 (+5 ms)Writing region close event to WAL at 1733811278710 (+62 ms)Running coprocessor post-close hooks at 1733811278714 (+4 ms)Closed at 1733811278715 (+1 ms) 2024-12-10T06:14:38,717 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:38,718 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=91cf7a8c24ad1cfeb344be5ce72aa8a5, regionState=CLOSED 2024-12-10T06:14:38,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:14:38,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-10T06:14:38,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 91cf7a8c24ad1cfeb344be5ce72aa8a5, server=c42c682b34f7,43983,1733811247584 in 1.4870 sec 2024-12-10T06:14:38,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-12-10T06:14:38,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91cf7a8c24ad1cfeb344be5ce72aa8a5, UNASSIGN in 1.4930 sec 2024-12-10T06:14:38,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-10T06:14:38,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5010 sec 2024-12-10T06:14:38,728 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811278728"}]},"ts":"1733811278728"} 2024-12-10T06:14:38,729 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T06:14:38,729 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T06:14:38,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5170 sec 2024-12-10T06:14:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T06:14:39,366 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:39,369 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T06:14:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:39,375 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T06:14:39,376 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:39,380 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:39,384 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/recovered.edits] 2024-12-10T06:14:39,388 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/e2a666f59f974a41846de586321ac860 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/e2a666f59f974a41846de586321ac860 2024-12-10T06:14:39,388 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/95d8689f1671442aa670fce5eb018d38 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/95d8689f1671442aa670fce5eb018d38 2024-12-10T06:14:39,388 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5a5d4af04a34ee7a1a73870dfd66185 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/A/f5a5d4af04a34ee7a1a73870dfd66185 2024-12-10T06:14:39,392 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/267893c0263a4128a8f99e0e094164dd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/267893c0263a4128a8f99e0e094164dd 2024-12-10T06:14:39,392 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3f2cfc5e508448daaa28fa64ab861b22 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3f2cfc5e508448daaa28fa64ab861b22 2024-12-10T06:14:39,392 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3ea4a173135443cc9a44b1a26a7d842a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/B/3ea4a173135443cc9a44b1a26a7d842a 2024-12-10T06:14:39,395 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/29ca1cdd7cbc41ed93b447a4f38042be to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/29ca1cdd7cbc41ed93b447a4f38042be 2024-12-10T06:14:39,395 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e2019d0d660441b3b2056ad634c85862 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/e2019d0d660441b3b2056ad634c85862 2024-12-10T06:14:39,396 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/68337b5a459a41df9b259eb4862e17fe to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/C/68337b5a459a41df9b259eb4862e17fe 2024-12-10T06:14:39,398 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/recovered.edits/457.seqid to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5/recovered.edits/457.seqid 2024-12-10T06:14:39,399 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/91cf7a8c24ad1cfeb344be5ce72aa8a5 2024-12-10T06:14:39,399 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T06:14:39,404 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:39,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-10T06:14:39,413 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T06:14:39,461 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T06:14:39,462 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:39,462 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T06:14:39,463 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733811279462"}]},"ts":"9223372036854775807"} 2024-12-10T06:14:39,467 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T06:14:39,467 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 91cf7a8c24ad1cfeb344be5ce72aa8a5, NAME => 'TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T06:14:39,467 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T06:14:39,467 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733811279467"}]},"ts":"9223372036854775807"} 2024-12-10T06:14:39,469 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T06:14:39,470 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:39,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-12-10T06:14:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T06:14:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T06:14:39,486 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:39,501 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=255 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_686407029_22 at /127.0.0.1:45222 [Waiting for operation #307] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c42c682b34f7:43983-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1197328417_22 at /127.0.0.1:37098 [Waiting for operation #296] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=422 (was 287) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7157 (was 7015) - AvailableMemoryMB LEAK? - 2024-12-10T06:14:39,513 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=255, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=422, ProcessCount=11, AvailableMemoryMB=7157 2024-12-10T06:14:39,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T06:14:39,515 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:14:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:39,517 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T06:14:39,517 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:39,517 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-12-10T06:14:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-10T06:14:39,518 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T06:14:39,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741936_1112 (size=1061) 2024-12-10T06:14:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-10T06:14:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-10T06:14:39,926 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6 2024-12-10T06:14:39,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741937_1113 (size=53) 2024-12-10T06:14:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-10T06:14:40,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:14:40,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9a8a17594d375f527e3c6e0f2fcae42e, disabling compactions & flushes 2024-12-10T06:14:40,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:40,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:40,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. after waiting 0 ms 2024-12-10T06:14:40,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:40,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:40,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9a8a17594d375f527e3c6e0f2fcae42e: Waiting for close lock at 1733811280333Disabling compacts and flushes for region at 1733811280333Disabling writes for close at 1733811280333Writing region close event to WAL at 1733811280333Closed at 1733811280333 2024-12-10T06:14:40,334 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T06:14:40,335 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733811280334"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733811280334"}]},"ts":"1733811280334"} 2024-12-10T06:14:40,336 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T06:14:40,337 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T06:14:40,338 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811280337"}]},"ts":"1733811280337"} 2024-12-10T06:14:40,339 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T06:14:40,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, ASSIGN}] 2024-12-10T06:14:40,341 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, ASSIGN 2024-12-10T06:14:40,342 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, ASSIGN; state=OFFLINE, location=c42c682b34f7,43983,1733811247584; forceNewPlan=false, retain=false 2024-12-10T06:14:40,493 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9a8a17594d375f527e3c6e0f2fcae42e, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:40,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, ASSIGN because future has completed 2024-12-10T06:14:40,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:14:40,651 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:40,651 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:14:40,652 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,652 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:14:40,652 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,652 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,653 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,654 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:14:40,655 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a8a17594d375f527e3c6e0f2fcae42e columnFamilyName A 2024-12-10T06:14:40,655 DEBUG [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-10T06:14:40,656 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(327): Store=9a8a17594d375f527e3c6e0f2fcae42e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:40,656 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,657 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:14:40,657 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a8a17594d375f527e3c6e0f2fcae42e columnFamilyName B 2024-12-10T06:14:40,657 DEBUG [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:40,658 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(327): Store=9a8a17594d375f527e3c6e0f2fcae42e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:40,658 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,659 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:14:40,659 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a8a17594d375f527e3c6e0f2fcae42e columnFamilyName C 2024-12-10T06:14:40,659 DEBUG [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:40,659 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(327): Store=9a8a17594d375f527e3c6e0f2fcae42e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:40,659 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:40,659 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,660 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,661 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,662 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,662 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,662 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:14:40,663 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,665 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:14:40,666 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 9a8a17594d375f527e3c6e0f2fcae42e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73866156, jitterRate=0.10069149732589722}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:14:40,666 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:40,667 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 9a8a17594d375f527e3c6e0f2fcae42e: Running coprocessor pre-open hook at 1733811280652Writing region info on filesystem at 1733811280652Initializing all the Stores at 1733811280653 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811280653Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811280653Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811280653Cleaning up temporary data from old regions at 1733811280662 (+9 ms)Running coprocessor post-open hooks at 1733811280666 (+4 ms)Region opened successfully at 1733811280667 (+1 ms) 2024-12-10T06:14:40,668 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., pid=30, masterSystemTime=1733811280648 2024-12-10T06:14:40,670 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:40,670 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:40,670 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9a8a17594d375f527e3c6e0f2fcae42e, regionState=OPEN, openSeqNum=2, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:40,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:14:40,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-10T06:14:40,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 in 178 msec 2024-12-10T06:14:40,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-10T06:14:40,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, ASSIGN in 335 msec 2024-12-10T06:14:40,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T06:14:40,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811280678"}]},"ts":"1733811280678"} 2024-12-10T06:14:40,680 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T06:14:40,681 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T06:14:40,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-12-10T06:14:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-10T06:14:41,666 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:41,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4db6c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:41,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:41,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:41,669 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c13cb7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:41,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ccf8029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:41,672 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:41,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:41,676 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T06:14:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c42c682b34f7,37415,1733811246834 2024-12-10T06:14:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@227cf768 2024-12-10T06:14:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T06:14:41,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T06:14:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T06:14:41,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:14:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T06:14:41,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741938_1114 (size=1097) 2024-12-10T06:14:42,100 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T06:14:42,100 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T06:14:42,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T06:14:42,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, REOPEN/MOVE}] 2024-12-10T06:14:42,113 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, REOPEN/MOVE 2024-12-10T06:14:42,114 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9a8a17594d375f527e3c6e0f2fcae42e, regionState=CLOSING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, REOPEN/MOVE because future has completed 2024-12-10T06:14:42,116 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:14:42,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:14:42,269 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 9a8a17594d375f527e3c6e0f2fcae42e, disabling compactions & flushes 2024-12-10T06:14:42,269 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. after waiting 0 ms 2024-12-10T06:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,273 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T06:14:42,274 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,274 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 9a8a17594d375f527e3c6e0f2fcae42e: Waiting for close lock at 1733811282269Running coprocessor pre-close hooks at 1733811282269Disabling compacts and flushes for region at 1733811282269Disabling writes for close at 1733811282269Writing region close event to WAL at 1733811282270 (+1 ms)Running coprocessor post-close hooks at 1733811282274 (+4 ms)Closed at 1733811282274 2024-12-10T06:14:42,274 WARN [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 9a8a17594d375f527e3c6e0f2fcae42e to self. 2024-12-10T06:14:42,276 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,277 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9a8a17594d375f527e3c6e0f2fcae42e, regionState=CLOSED 2024-12-10T06:14:42,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:14:42,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-10T06:14:42,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 in 163 msec 2024-12-10T06:14:42,282 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, REOPEN/MOVE; state=CLOSED, location=c42c682b34f7,43983,1733811247584; forceNewPlan=false, retain=true 2024-12-10T06:14:42,432 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9a8a17594d375f527e3c6e0f2fcae42e, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, REOPEN/MOVE because future has completed 2024-12-10T06:14:42,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:14:42,590 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,590 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:14:42,590 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,590 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:14:42,591 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,591 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,593 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,593 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:14:42,598 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a8a17594d375f527e3c6e0f2fcae42e columnFamilyName A 2024-12-10T06:14:42,600 DEBUG [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:42,600 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(327): Store=9a8a17594d375f527e3c6e0f2fcae42e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:42,601 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,601 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:14:42,602 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a8a17594d375f527e3c6e0f2fcae42e columnFamilyName B 2024-12-10T06:14:42,602 DEBUG [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:42,602 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(327): Store=9a8a17594d375f527e3c6e0f2fcae42e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:42,602 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,603 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:14:42,603 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a8a17594d375f527e3c6e0f2fcae42e columnFamilyName C 2024-12-10T06:14:42,603 DEBUG [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:42,603 INFO [StoreOpener-9a8a17594d375f527e3c6e0f2fcae42e-1 {}] regionserver.HStore(327): Store=9a8a17594d375f527e3c6e0f2fcae42e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:14:42,603 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,603 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,604 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,605 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,606 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,606 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,606 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:14:42,608 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,608 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 9a8a17594d375f527e3c6e0f2fcae42e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72306904, jitterRate=0.07745683193206787}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:14:42,609 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,611 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 9a8a17594d375f527e3c6e0f2fcae42e: Running coprocessor pre-open hook at 1733811282591Writing region info on filesystem at 1733811282591Initializing all the Stores at 1733811282591Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811282591Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811282592 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811282592Cleaning up temporary data from old regions at 1733811282606 (+14 ms)Running coprocessor post-open hooks at 1733811282609 (+3 ms)Region opened successfully at 1733811282611 (+2 ms) 2024-12-10T06:14:42,612 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., pid=35, masterSystemTime=1733811282586 2024-12-10T06:14:42,614 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,614 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,614 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9a8a17594d375f527e3c6e0f2fcae42e, regionState=OPEN, openSeqNum=5, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:14:42,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-12-10T06:14:42,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 in 182 msec 2024-12-10T06:14:42,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-10T06:14:42,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, REOPEN/MOVE in 507 msec 2024-12-10T06:14:42,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-10T06:14:42,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-12-10T06:14:42,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 941 msec 2024-12-10T06:14:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-10T06:14:42,630 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:42,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:42,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:42,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58dc279b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:42,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a33850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:42,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:42,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15684a5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:42,636 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119252c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:42,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:42,639 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598d73a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:42,640 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce0a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:42,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:42,642 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a5637c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:42,644 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ef9b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:42,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:42,646 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51484c31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:42,647 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64158c90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:42,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:42,649 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f26158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:42,650 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d64e20d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:42,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:42,652 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295e4063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:42,653 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771a0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:42,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:42,655 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c7e2da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:42,656 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22644b0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2874ba29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:14:42,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:14:42,658 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:14:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:14:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:14:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7428d991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:14:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:14:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:14:42,659 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:14:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dfab01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:14:42,661 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:42,661 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:42,661 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:42,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:42,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:42,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:42,666 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:42,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:42,668 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-10T06:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:14:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:14:42,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:42,670 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T06:14:42,671 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:14:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:14:42,672 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:42,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:42,673 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:42,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:42,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:14:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:14:42,679 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:42,680 DEBUG [Thread-588 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:42,680 DEBUG [Thread-588 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:42,680 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:14:42,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:42,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:14:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:14:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:14:42,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:14:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:14:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:14:42,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T06:14:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:42,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121092903f2f96b64500b7538192611dd52b_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811282695/Put/seqid=0 2024-12-10T06:14:42,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741939_1115 (size=14594) 2024-12-10T06:14:42,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811342756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811342760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811342763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811342764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T06:14:42,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811342766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T06:14:42,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:42,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:42,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:42,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811342865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811342886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811342886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811342886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811342886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:42,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T06:14:42,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:42,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:42,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:42,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T06:14:43,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811343076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811343096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811343096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811343096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811343096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T06:14:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121092903f2f96b64500b7538192611dd52b_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121092903f2f96b64500b7538192611dd52b_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:43,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/9cbbf7bc60464ebe93647de55500b860, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:43,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/9cbbf7bc60464ebe93647de55500b860 is 175, key is test_row_0/A:col10/1733811282695/Put/seqid=0 2024-12-10T06:14:43,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741940_1116 (size=39549) 2024-12-10T06:14:43,191 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/9cbbf7bc60464ebe93647de55500b860 2024-12-10T06:14:43,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/9be4bd475fe4452080df1446b33c7059 is 50, key is test_row_0/B:col10/1733811282695/Put/seqid=0 2024-12-10T06:14:43,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741941_1117 (size=12001) 2024-12-10T06:14:43,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/9be4bd475fe4452080df1446b33c7059 2024-12-10T06:14:43,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/f660f545d09c47df9b9a126b9de38bd0 is 50, key is test_row_0/C:col10/1733811282695/Put/seqid=0 2024-12-10T06:14:43,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741942_1118 (size=12001) 2024-12-10T06:14:43,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T06:14:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T06:14:43,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811343395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811343406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811343406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811343406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811343406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:43,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T06:14:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T06:14:43,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:43,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:43,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/f660f545d09c47df9b9a126b9de38bd0 2024-12-10T06:14:43,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/9cbbf7bc60464ebe93647de55500b860 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9cbbf7bc60464ebe93647de55500b860 2024-12-10T06:14:43,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9cbbf7bc60464ebe93647de55500b860, entries=200, sequenceid=17, filesize=38.6 K 2024-12-10T06:14:43,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/9be4bd475fe4452080df1446b33c7059 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/9be4bd475fe4452080df1446b33c7059 2024-12-10T06:14:43,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/9be4bd475fe4452080df1446b33c7059, entries=150, sequenceid=17, filesize=11.7 K 2024-12-10T06:14:43,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/f660f545d09c47df9b9a126b9de38bd0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f660f545d09c47df9b9a126b9de38bd0 2024-12-10T06:14:43,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f660f545d09c47df9b9a126b9de38bd0, entries=150, sequenceid=17, filesize=11.7 K 2024-12-10T06:14:43,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 9a8a17594d375f527e3c6e0f2fcae42e in 1032ms, sequenceid=17, compaction requested=false 2024-12-10T06:14:43,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:43,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-10T06:14:43,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,753 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T06:14:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:43,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121002377bdef5334f179813718ddfa110fb_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811282759/Put/seqid=0 2024-12-10T06:14:43,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741943_1119 (size=12154) 2024-12-10T06:14:43,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,792 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121002377bdef5334f179813718ddfa110fb_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121002377bdef5334f179813718ddfa110fb_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:43,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/b917f9a138824e1cbd0cd30a5c73d1ce, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:43,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/b917f9a138824e1cbd0cd30a5c73d1ce is 175, key is test_row_0/A:col10/1733811282759/Put/seqid=0 2024-12-10T06:14:43,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741944_1120 (size=30955) 2024-12-10T06:14:43,803 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=37, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/b917f9a138824e1cbd0cd30a5c73d1ce 2024-12-10T06:14:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T06:14:43,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/5a65a4dcd6564952abfe365b8f19921e is 50, key is test_row_0/B:col10/1733811282759/Put/seqid=0 2024-12-10T06:14:43,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741945_1121 (size=12001) 2024-12-10T06:14:43,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/5a65a4dcd6564952abfe365b8f19921e 2024-12-10T06:14:43,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/8ff9781adb5347e0a39438900d99d876 is 50, key is test_row_0/C:col10/1733811282759/Put/seqid=0 2024-12-10T06:14:43,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741946_1122 (size=12001) 2024-12-10T06:14:43,858 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/8ff9781adb5347e0a39438900d99d876 2024-12-10T06:14:43,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/b917f9a138824e1cbd0cd30a5c73d1ce as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b917f9a138824e1cbd0cd30a5c73d1ce 2024-12-10T06:14:43,876 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b917f9a138824e1cbd0cd30a5c73d1ce, entries=150, sequenceid=37, filesize=30.2 K 2024-12-10T06:14:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/5a65a4dcd6564952abfe365b8f19921e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/5a65a4dcd6564952abfe365b8f19921e 2024-12-10T06:14:43,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/5a65a4dcd6564952abfe365b8f19921e, entries=150, sequenceid=37, filesize=11.7 K 2024-12-10T06:14:43,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/8ff9781adb5347e0a39438900d99d876 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8ff9781adb5347e0a39438900d99d876 2024-12-10T06:14:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,902 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8ff9781adb5347e0a39438900d99d876, entries=150, sequenceid=37, filesize=11.7 K 2024-12-10T06:14:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 9a8a17594d375f527e3c6e0f2fcae42e in 150ms, sequenceid=37, compaction requested=false 2024-12-10T06:14:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-10T06:14:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-10T06:14:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-10T06:14:43,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2330 sec 2024-12-10T06:14:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.2410 sec 2024-12-10T06:14:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b08faf8c0b5e4d8fb680bedfacef9d6e_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811283954/Put/seqid=0 2024-12-10T06:14:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741947_1123 (size=21918) 2024-12-10T06:14:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,023 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,028 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b08faf8c0b5e4d8fb680bedfacef9d6e_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b08faf8c0b5e4d8fb680bedfacef9d6e_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,030 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/f632278a6b0e4a93b5f9879918e0fd81, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/f632278a6b0e4a93b5f9879918e0fd81 is 175, key is test_row_0/A:col10/1733811283954/Put/seqid=0 2024-12-10T06:14:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741948_1124 (size=65323) 2024-12-10T06:14:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,037 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=47, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/f632278a6b0e4a93b5f9879918e0fd81 2024-12-10T06:14:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733811283964, max timestamp=1733811284032Num uniques 859; 2024-12-10T06:14:44,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733811283964, max timestamp=1733811284032Num uniques 859; 2024-12-10T06:14:44,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733811283964, max timestamp=1733811284032Num uniques 859; 2024-12-10T06:14:44,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-10T06:14:44,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-10T06:14:44,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-10T06:14:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/e210880f98cd4f5b906b9cae1d521ad6 is 50, key is test_row_0/B:col10/1733811283954/Put/seqid=0 2024-12-10T06:14:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811344046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811344048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811344051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811344053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811344056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741949_1125 (size=21365) 2024-12-10T06:14:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/e210880f98cd4f5b906b9cae1d521ad6 2024-12-10T06:14:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/3a7a98c65519457abef9e5a62d55b8e2 is 50, key is test_row_0/C:col10/1733811283954/Put/seqid=0 2024-12-10T06:14:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741950_1126 (size=21365) 2024-12-10T06:14:44,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/3a7a98c65519457abef9e5a62d55b8e2 2024-12-10T06:14:44,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/f632278a6b0e4a93b5f9879918e0fd81 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/f632278a6b0e4a93b5f9879918e0fd81 2024-12-10T06:14:44,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/f632278a6b0e4a93b5f9879918e0fd81, entries=350, sequenceid=47, filesize=63.8 K 2024-12-10T06:14:44,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/e210880f98cd4f5b906b9cae1d521ad6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e210880f98cd4f5b906b9cae1d521ad6 2024-12-10T06:14:44,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e210880f98cd4f5b906b9cae1d521ad6, entries=350, sequenceid=47, filesize=20.9 K 2024-12-10T06:14:44,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/3a7a98c65519457abef9e5a62d55b8e2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/3a7a98c65519457abef9e5a62d55b8e2 2024-12-10T06:14:44,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/3a7a98c65519457abef9e5a62d55b8e2, entries=350, sequenceid=47, filesize=20.9 K 2024-12-10T06:14:44,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 9a8a17594d375f527e3c6e0f2fcae42e in 181ms, sequenceid=47, compaction requested=true 2024-12-10T06:14:44,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:44,145 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:44,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:44,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:44,145 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:44,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:44,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:44,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:44,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:44,147 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:44,147 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/A is initiating minor compaction (all files) 2024-12-10T06:14:44,147 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/A in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:44,147 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9cbbf7bc60464ebe93647de55500b860, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b917f9a138824e1cbd0cd30a5c73d1ce, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/f632278a6b0e4a93b5f9879918e0fd81] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=132.6 K 2024-12-10T06:14:44,147 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:44,148 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9cbbf7bc60464ebe93647de55500b860, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b917f9a138824e1cbd0cd30a5c73d1ce, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/f632278a6b0e4a93b5f9879918e0fd81] 2024-12-10T06:14:44,149 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:44,149 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/B is initiating minor compaction (all files) 2024-12-10T06:14:44,149 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/B in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:44,149 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/9be4bd475fe4452080df1446b33c7059, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/5a65a4dcd6564952abfe365b8f19921e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e210880f98cd4f5b906b9cae1d521ad6] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=44.3 K 2024-12-10T06:14:44,150 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be4bd475fe4452080df1446b33c7059, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733811282680 2024-12-10T06:14:44,150 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cbbf7bc60464ebe93647de55500b860, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733811282680 2024-12-10T06:14:44,150 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a65a4dcd6564952abfe365b8f19921e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733811282731 2024-12-10T06:14:44,150 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting b917f9a138824e1cbd0cd30a5c73d1ce, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733811282731 2024-12-10T06:14:44,151 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting e210880f98cd4f5b906b9cae1d521ad6, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733811283948 2024-12-10T06:14:44,151 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f632278a6b0e4a93b5f9879918e0fd81, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733811283948 2024-12-10T06:14:44,168 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#B#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:44,169 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/17e63a699a0b4e83ade74042127a4ce9 is 50, key is test_row_0/B:col10/1733811283954/Put/seqid=0 2024-12-10T06:14:44,173 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:44,182 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412104fd447e701e74df4aa8b92ae5f8b9fa3_9a8a17594d375f527e3c6e0f2fcae42e store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:44,186 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412104fd447e701e74df4aa8b92ae5f8b9fa3_9a8a17594d375f527e3c6e0f2fcae42e, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:44,186 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104fd447e701e74df4aa8b92ae5f8b9fa3_9a8a17594d375f527e3c6e0f2fcae42e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:44,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741951_1127 (size=12104) 2024-12-10T06:14:44,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741952_1128 (size=4469) 2024-12-10T06:14:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:44,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=174.43 KB heapSize=438.89 KB 2024-12-10T06:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:44,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811344216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811344218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811344218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103a8da956085848468b26462edf1f8fdd_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811284214/Put/seqid=0 2024-12-10T06:14:44,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811344226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811344227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741953_1129 (size=17034) 2024-12-10T06:14:44,253 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:14:44,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811344335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811344335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811344335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811344346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811344346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811344545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811344546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811344546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811344565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811344565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,590 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/17e63a699a0b4e83ade74042127a4ce9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/17e63a699a0b4e83ade74042127a4ce9 2024-12-10T06:14:44,596 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#A#compaction#109 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:44,596 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/B of 9a8a17594d375f527e3c6e0f2fcae42e into 17e63a699a0b4e83ade74042127a4ce9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:44,596 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:44,596 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/3e51f3cf46a543bd96676d5603967063 is 175, key is test_row_0/A:col10/1733811283954/Put/seqid=0 2024-12-10T06:14:44,597 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/B, priority=13, startTime=1733811284145; duration=0sec 2024-12-10T06:14:44,597 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:44,597 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:B 2024-12-10T06:14:44,597 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:44,598 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:44,598 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/C is initiating minor compaction (all files) 2024-12-10T06:14:44,599 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/C in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:44,599 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f660f545d09c47df9b9a126b9de38bd0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8ff9781adb5347e0a39438900d99d876, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/3a7a98c65519457abef9e5a62d55b8e2] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=44.3 K 2024-12-10T06:14:44,600 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting f660f545d09c47df9b9a126b9de38bd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733811282680 2024-12-10T06:14:44,600 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ff9781adb5347e0a39438900d99d876, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733811282731 2024-12-10T06:14:44,600 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a7a98c65519457abef9e5a62d55b8e2, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733811283948 2024-12-10T06:14:44,619 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#C#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:44,620 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/45fe9d855f834ada819708b826a86c53 is 50, key is test_row_0/C:col10/1733811283954/Put/seqid=0 2024-12-10T06:14:44,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741954_1130 (size=31058) 2024-12-10T06:14:44,629 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/3e51f3cf46a543bd96676d5603967063 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/3e51f3cf46a543bd96676d5603967063 2024-12-10T06:14:44,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741955_1131 (size=12104) 2024-12-10T06:14:44,637 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/A of 9a8a17594d375f527e3c6e0f2fcae42e into 3e51f3cf46a543bd96676d5603967063(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:44,637 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:44,637 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/A, priority=13, startTime=1733811284144; duration=0sec 2024-12-10T06:14:44,637 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:44,637 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:A 2024-12-10T06:14:44,643 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/45fe9d855f834ada819708b826a86c53 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/45fe9d855f834ada819708b826a86c53 2024-12-10T06:14:44,652 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,653 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/C of 9a8a17594d375f527e3c6e0f2fcae42e into 45fe9d855f834ada819708b826a86c53(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:44,653 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:44,653 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/C, priority=13, startTime=1733811284145; duration=0sec 2024-12-10T06:14:44,654 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:44,654 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:C 2024-12-10T06:14:44,658 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103a8da956085848468b26462edf1f8fdd_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103a8da956085848468b26462edf1f8fdd_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:44,661 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ff35cfe62eff4aaf9fd76f80b34c23e3, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:44,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ff35cfe62eff4aaf9fd76f80b34c23e3 is 175, key is test_row_0/A:col10/1733811284214/Put/seqid=0 2024-12-10T06:14:44,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741956_1132 (size=48139) 2024-12-10T06:14:44,668 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ff35cfe62eff4aaf9fd76f80b34c23e3 2024-12-10T06:14:44,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/3578e82e9c58407db7ec6d5407425dd2 is 50, key is test_row_0/B:col10/1733811284214/Put/seqid=0 2024-12-10T06:14:44,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741957_1133 (size=12001) 2024-12-10T06:14:44,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/3578e82e9c58407db7ec6d5407425dd2 2024-12-10T06:14:44,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/8328d8038b244b29af3431654fe8733b is 50, key is test_row_0/C:col10/1733811284214/Put/seqid=0 2024-12-10T06:14:44,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741958_1134 (size=12001) 2024-12-10T06:14:44,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/8328d8038b244b29af3431654fe8733b 2024-12-10T06:14:44,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ff35cfe62eff4aaf9fd76f80b34c23e3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ff35cfe62eff4aaf9fd76f80b34c23e3 2024-12-10T06:14:44,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ff35cfe62eff4aaf9fd76f80b34c23e3, entries=250, sequenceid=78, filesize=47.0 K 2024-12-10T06:14:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/3578e82e9c58407db7ec6d5407425dd2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3578e82e9c58407db7ec6d5407425dd2 2024-12-10T06:14:44,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3578e82e9c58407db7ec6d5407425dd2, entries=150, sequenceid=78, filesize=11.7 K 2024-12-10T06:14:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/8328d8038b244b29af3431654fe8733b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8328d8038b244b29af3431654fe8733b 2024-12-10T06:14:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8328d8038b244b29af3431654fe8733b, entries=150, sequenceid=78, filesize=11.7 K 2024-12-10T06:14:44,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~478.69 KB/490176, currentSize=13.42 KB/13740 for 9a8a17594d375f527e3c6e0f2fcae42e in 534ms, sequenceid=78, compaction requested=false 2024-12-10T06:14:44,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-10T06:14:44,815 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-10T06:14:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T06:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,819 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:44,820 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:44,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210845b2ae6e2ef4672b0238d1cfa8dcda2_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811284867/Put/seqid=0 2024-12-10T06:14:44,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:44,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741959_1135 (size=21918) 2024-12-10T06:14:44,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733811284898, max timestamp=1733811284920Num uniques 859; 2024-12-10T06:14:44,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733811284898, max timestamp=1733811284920Num uniques 859; 2024-12-10T06:14:44,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T06:14:44,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733811284898, max timestamp=1733811284920Num uniques 859; 2024-12-10T06:14:44,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T06:14:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T06:14:44,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T06:14:44,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811344950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811344961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811344963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811344966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811344966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:44,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T06:14:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:44,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:44,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811345075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811345075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811345076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811345076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811345076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T06:14:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T06:14:45,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T06:14:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811345285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811345286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811345286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811345287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811345295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,307 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:45,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210845b2ae6e2ef4672b0238d1cfa8dcda2_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210845b2ae6e2ef4672b0238d1cfa8dcda2_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:45,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/7747553791bd4adfa2c84e7eb6e43292, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:45,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/7747553791bd4adfa2c84e7eb6e43292 is 175, key is test_row_0/A:col10/1733811284867/Put/seqid=0 2024-12-10T06:14:45,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741960_1136 (size=65323) 2024-12-10T06:14:45,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/7747553791bd4adfa2c84e7eb6e43292 2024-12-10T06:14:45,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/4ac51d6c716545dfafc15aedb582bbb0 is 50, key is test_row_0/B:col10/1733811284867/Put/seqid=0 2024-12-10T06:14:45,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741961_1137 (size=12001) 2024-12-10T06:14:45,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T06:14:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T06:14:45,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T06:14:45,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:45,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811345605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811345605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811345606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811345606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811345606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:45,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T06:14:45,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:45,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/4ac51d6c716545dfafc15aedb582bbb0 2024-12-10T06:14:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/ad76fda382e34fe887834bf00a0e1b03 is 50, key is test_row_0/C:col10/1733811284867/Put/seqid=0 2024-12-10T06:14:45,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741962_1138 (size=12001) 2024-12-10T06:14:45,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T06:14:45,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:45,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:45,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T06:14:46,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T06:14:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:46,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:46,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811346116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:46,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:46,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811346116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:46,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811346125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:46,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:46,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811346125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811346125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:46,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/ad76fda382e34fe887834bf00a0e1b03 2024-12-10T06:14:46,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/7747553791bd4adfa2c84e7eb6e43292 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/7747553791bd4adfa2c84e7eb6e43292 2024-12-10T06:14:46,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/7747553791bd4adfa2c84e7eb6e43292, entries=350, sequenceid=91, filesize=63.8 K 2024-12-10T06:14:46,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/4ac51d6c716545dfafc15aedb582bbb0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/4ac51d6c716545dfafc15aedb582bbb0 2024-12-10T06:14:46,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/4ac51d6c716545dfafc15aedb582bbb0, entries=150, sequenceid=91, filesize=11.7 K 2024-12-10T06:14:46,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/ad76fda382e34fe887834bf00a0e1b03 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ad76fda382e34fe887834bf00a0e1b03 2024-12-10T06:14:46,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ad76fda382e34fe887834bf00a0e1b03, entries=150, sequenceid=91, filesize=11.7 K 2024-12-10T06:14:46,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 9a8a17594d375f527e3c6e0f2fcae42e in 1315ms, sequenceid=91, compaction requested=true 2024-12-10T06:14:46,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:46,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:46,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:46,184 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:46,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:46,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:46,184 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:46,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:46,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:46,185 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144520 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:46,185 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:46,185 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/A is initiating minor compaction (all files) 2024-12-10T06:14:46,185 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/B is initiating minor compaction (all files) 2024-12-10T06:14:46,185 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/A in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:46,185 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/B in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:46,185 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/3e51f3cf46a543bd96676d5603967063, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ff35cfe62eff4aaf9fd76f80b34c23e3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/7747553791bd4adfa2c84e7eb6e43292] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=141.1 K 2024-12-10T06:14:46,185 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/17e63a699a0b4e83ade74042127a4ce9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3578e82e9c58407db7ec6d5407425dd2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/4ac51d6c716545dfafc15aedb582bbb0] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=35.3 K 2024-12-10T06:14:46,185 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:46,186 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/3e51f3cf46a543bd96676d5603967063, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ff35cfe62eff4aaf9fd76f80b34c23e3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/7747553791bd4adfa2c84e7eb6e43292] 2024-12-10T06:14:46,186 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 17e63a699a0b4e83ade74042127a4ce9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733811283954 2024-12-10T06:14:46,186 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e51f3cf46a543bd96676d5603967063, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733811283954 2024-12-10T06:14:46,187 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 3578e82e9c58407db7ec6d5407425dd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733811284194 2024-12-10T06:14:46,187 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff35cfe62eff4aaf9fd76f80b34c23e3, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733811284053 2024-12-10T06:14:46,187 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac51d6c716545dfafc15aedb582bbb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733811284861 2024-12-10T06:14:46,187 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7747553791bd4adfa2c84e7eb6e43292, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733811284223 2024-12-10T06:14:46,197 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:46,201 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#B#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:46,201 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/cec5a6de9c9a43ddb77ed596bf98db42 is 50, key is test_row_0/B:col10/1733811284867/Put/seqid=0 2024-12-10T06:14:46,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T06:14:46,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:46,203 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210ee6e87c92c3c42038528167b5b68a95a_9a8a17594d375f527e3c6e0f2fcae42e store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:46,203 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-10T06:14:46,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:46,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:46,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:46,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:46,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:46,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:46,206 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210ee6e87c92c3c42038528167b5b68a95a_9a8a17594d375f527e3c6e0f2fcae42e, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:46,207 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ee6e87c92c3c42038528167b5b68a95a_9a8a17594d375f527e3c6e0f2fcae42e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:46,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741963_1139 (size=12207) 2024-12-10T06:14:46,215 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/cec5a6de9c9a43ddb77ed596bf98db42 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/cec5a6de9c9a43ddb77ed596bf98db42 2024-12-10T06:14:46,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f11fd5b11594409680bc2ff33600eb5a_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811284963/Put/seqid=0 2024-12-10T06:14:46,229 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/B of 9a8a17594d375f527e3c6e0f2fcae42e into cec5a6de9c9a43ddb77ed596bf98db42(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:46,229 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:46,229 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/B, priority=13, startTime=1733811286184; duration=0sec 2024-12-10T06:14:46,230 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:46,230 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:B 2024-12-10T06:14:46,230 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:46,231 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:46,232 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/C is initiating minor compaction (all files) 2024-12-10T06:14:46,232 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/C in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:46,232 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/45fe9d855f834ada819708b826a86c53, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8328d8038b244b29af3431654fe8733b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ad76fda382e34fe887834bf00a0e1b03] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=35.3 K 2024-12-10T06:14:46,232 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 45fe9d855f834ada819708b826a86c53, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733811283954 2024-12-10T06:14:46,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741964_1140 (size=4469) 2024-12-10T06:14:46,234 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8328d8038b244b29af3431654fe8733b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733811284194 2024-12-10T06:14:46,235 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ad76fda382e34fe887834bf00a0e1b03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733811284861 2024-12-10T06:14:46,235 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#A#compaction#117 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:46,236 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/8c89af72308a43b4b43f41379d0776f5 is 175, key is test_row_0/A:col10/1733811284867/Put/seqid=0 2024-12-10T06:14:46,246 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#C#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:46,247 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/72d00a783e7646169bf4a8b64f6a5d48 is 50, key is test_row_0/C:col10/1733811284867/Put/seqid=0 2024-12-10T06:14:46,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741966_1142 (size=31161) 2024-12-10T06:14:46,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741965_1141 (size=12154) 2024-12-10T06:14:46,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:46,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741967_1143 (size=12207) 2024-12-10T06:14:46,270 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f11fd5b11594409680bc2ff33600eb5a_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f11fd5b11594409680bc2ff33600eb5a_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:46,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/5fa89e3300774625b46ef86fa97d3887, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:46,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/5fa89e3300774625b46ef86fa97d3887 is 175, key is test_row_0/A:col10/1733811284963/Put/seqid=0 2024-12-10T06:14:46,276 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/72d00a783e7646169bf4a8b64f6a5d48 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/72d00a783e7646169bf4a8b64f6a5d48 2024-12-10T06:14:46,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741968_1144 (size=30955) 2024-12-10T06:14:46,282 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/C of 9a8a17594d375f527e3c6e0f2fcae42e into 72d00a783e7646169bf4a8b64f6a5d48(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:46,282 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:46,282 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/C, priority=13, startTime=1733811286184; duration=0sec 2024-12-10T06:14:46,282 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:46,282 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:C 2024-12-10T06:14:46,658 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/8c89af72308a43b4b43f41379d0776f5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/8c89af72308a43b4b43f41379d0776f5 2024-12-10T06:14:46,666 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/A of 9a8a17594d375f527e3c6e0f2fcae42e into 8c89af72308a43b4b43f41379d0776f5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:46,666 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:46,666 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/A, priority=13, startTime=1733811286184; duration=0sec 2024-12-10T06:14:46,666 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:46,666 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:A 2024-12-10T06:14:46,678 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/5fa89e3300774625b46ef86fa97d3887 2024-12-10T06:14:46,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/82fa0ad53f134891be98a7c77e31d0e9 is 50, key is test_row_0/B:col10/1733811284963/Put/seqid=0 2024-12-10T06:14:46,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741969_1145 (size=12001) 2024-12-10T06:14:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T06:14:47,093 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/82fa0ad53f134891be98a7c77e31d0e9 2024-12-10T06:14:47,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/565b19fe99b34d3b9ffafe9cf04a24e5 is 50, key is test_row_0/C:col10/1733811284963/Put/seqid=0 2024-12-10T06:14:47,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741970_1146 (size=12001) 2024-12-10T06:14:47,117 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/565b19fe99b34d3b9ffafe9cf04a24e5 2024-12-10T06:14:47,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/5fa89e3300774625b46ef86fa97d3887 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5fa89e3300774625b46ef86fa97d3887 2024-12-10T06:14:47,131 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5fa89e3300774625b46ef86fa97d3887, entries=150, sequenceid=127, filesize=30.2 K 2024-12-10T06:14:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:47,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/82fa0ad53f134891be98a7c77e31d0e9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/82fa0ad53f134891be98a7c77e31d0e9 2024-12-10T06:14:47,141 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/82fa0ad53f134891be98a7c77e31d0e9, entries=150, sequenceid=127, filesize=11.7 K 2024-12-10T06:14:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/565b19fe99b34d3b9ffafe9cf04a24e5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/565b19fe99b34d3b9ffafe9cf04a24e5 2024-12-10T06:14:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/565b19fe99b34d3b9ffafe9cf04a24e5, entries=150, sequenceid=127, filesize=11.7 K 2024-12-10T06:14:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,151 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=20.13 KB/20610 for 9a8a17594d375f527e3c6e0f2fcae42e in 948ms, sequenceid=127, compaction requested=false 2024-12-10T06:14:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-10T06:14:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-10T06:14:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-10T06:14:47,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-12-10T06:14:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.3420 sec 2024-12-10T06:14:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:47,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T06:14:47,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:47,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:47,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:47,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:47,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:47,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:47,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104869c5d4992248cc8c1ac5c653e0c61f_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811287183/Put/seqid=0 2024-12-10T06:14:47,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741971_1147 (size=12304) 2024-12-10T06:14:47,212 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:47,219 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104869c5d4992248cc8c1ac5c653e0c61f_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104869c5d4992248cc8c1ac5c653e0c61f_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:47,221 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ebd4097964f64eb9b56f7f7274d17125, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:47,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ebd4097964f64eb9b56f7f7274d17125 is 175, key is test_row_0/A:col10/1733811287183/Put/seqid=0 2024-12-10T06:14:47,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741972_1148 (size=31105) 2024-12-10T06:14:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811347220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733811287206, max timestamp=1733811287229Num uniques 859; 2024-12-10T06:14:47,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T06:14:47,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733811287206, max timestamp=1733811287229Num uniques 859; 2024-12-10T06:14:47,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T06:14:47,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733811287206, max timestamp=1733811287229Num uniques 859; 2024-12-10T06:14:47,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T06:14:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811347222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811347274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811347274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811347274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811347345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811347345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811347385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811347385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811347385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811347555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811347555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811347595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811347596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811347596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,631 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ebd4097964f64eb9b56f7f7274d17125 2024-12-10T06:14:47,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/3faf0c21622940b9830d567e3370239e is 50, key is test_row_0/B:col10/1733811287183/Put/seqid=0 2024-12-10T06:14:47,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741973_1149 (size=12151) 2024-12-10T06:14:47,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/3faf0c21622940b9830d567e3370239e 2024-12-10T06:14:47,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/01246ab14f3b4349ad36c53e0634824e is 50, key is test_row_0/C:col10/1733811287183/Put/seqid=0 2024-12-10T06:14:47,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741974_1150 (size=12151) 2024-12-10T06:14:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811347865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811347865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811347905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811347906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811347906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/01246ab14f3b4349ad36c53e0634824e 2024-12-10T06:14:48,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ebd4097964f64eb9b56f7f7274d17125 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ebd4097964f64eb9b56f7f7274d17125 2024-12-10T06:14:48,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ebd4097964f64eb9b56f7f7274d17125, entries=150, sequenceid=141, filesize=30.4 K 2024-12-10T06:14:48,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/3faf0c21622940b9830d567e3370239e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3faf0c21622940b9830d567e3370239e 2024-12-10T06:14:48,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3faf0c21622940b9830d567e3370239e, entries=150, sequenceid=141, filesize=11.9 K 2024-12-10T06:14:48,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/01246ab14f3b4349ad36c53e0634824e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/01246ab14f3b4349ad36c53e0634824e 2024-12-10T06:14:48,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/01246ab14f3b4349ad36c53e0634824e, entries=150, sequenceid=141, filesize=11.9 K 2024-12-10T06:14:48,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 9a8a17594d375f527e3c6e0f2fcae42e in 899ms, sequenceid=141, compaction requested=true 2024-12-10T06:14:48,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:48,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:48,088 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:48,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:48,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:48,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:48,089 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:48,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:48,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:48,090 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:48,090 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/A is initiating minor compaction (all files) 2024-12-10T06:14:48,090 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/A in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:48,090 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/8c89af72308a43b4b43f41379d0776f5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5fa89e3300774625b46ef86fa97d3887, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ebd4097964f64eb9b56f7f7274d17125] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=91.0 K 2024-12-10T06:14:48,090 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:48,090 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/8c89af72308a43b4b43f41379d0776f5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5fa89e3300774625b46ef86fa97d3887, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ebd4097964f64eb9b56f7f7274d17125] 2024-12-10T06:14:48,091 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:48,091 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/B is initiating minor compaction (all files) 2024-12-10T06:14:48,091 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/B in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:48,091 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/cec5a6de9c9a43ddb77ed596bf98db42, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/82fa0ad53f134891be98a7c77e31d0e9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3faf0c21622940b9830d567e3370239e] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=35.5 K 2024-12-10T06:14:48,092 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c89af72308a43b4b43f41379d0776f5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733811284861 2024-12-10T06:14:48,092 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting cec5a6de9c9a43ddb77ed596bf98db42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733811284861 2024-12-10T06:14:48,092 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fa89e3300774625b46ef86fa97d3887, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733811284960 2024-12-10T06:14:48,092 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 82fa0ad53f134891be98a7c77e31d0e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733811284960 2024-12-10T06:14:48,093 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebd4097964f64eb9b56f7f7274d17125, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811287183 2024-12-10T06:14:48,093 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 3faf0c21622940b9830d567e3370239e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811287183 2024-12-10T06:14:48,105 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:48,106 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:48,107 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/aaf15bbc329a40be9740c49ba55c52de is 50, key is test_row_0/B:col10/1733811287183/Put/seqid=0 2024-12-10T06:14:48,108 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210d7771d05e8754dd8b7275d5286d2f557_9a8a17594d375f527e3c6e0f2fcae42e store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:48,111 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210d7771d05e8754dd8b7275d5286d2f557_9a8a17594d375f527e3c6e0f2fcae42e, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:48,111 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d7771d05e8754dd8b7275d5286d2f557_9a8a17594d375f527e3c6e0f2fcae42e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:48,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741975_1151 (size=12459) 2024-12-10T06:14:48,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741976_1152 (size=4469) 2024-12-10T06:14:48,125 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#A#compaction#127 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:48,126 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/88d6bb788555412da75207c8b83b6ca3 is 175, key is test_row_0/A:col10/1733811287183/Put/seqid=0 2024-12-10T06:14:48,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741977_1153 (size=31413) 2024-12-10T06:14:48,139 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/88d6bb788555412da75207c8b83b6ca3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/88d6bb788555412da75207c8b83b6ca3 2024-12-10T06:14:48,154 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/A of 9a8a17594d375f527e3c6e0f2fcae42e into 88d6bb788555412da75207c8b83b6ca3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:48,154 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:48,154 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/A, priority=13, startTime=1733811288088; duration=0sec 2024-12-10T06:14:48,154 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:48,154 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:A 2024-12-10T06:14:48,154 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:48,155 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:48,155 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/C is initiating minor compaction (all files) 2024-12-10T06:14:48,156 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/C in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:48,156 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/72d00a783e7646169bf4a8b64f6a5d48, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/565b19fe99b34d3b9ffafe9cf04a24e5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/01246ab14f3b4349ad36c53e0634824e] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=35.5 K 2024-12-10T06:14:48,156 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72d00a783e7646169bf4a8b64f6a5d48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733811284861 2024-12-10T06:14:48,157 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 565b19fe99b34d3b9ffafe9cf04a24e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733811284960 2024-12-10T06:14:48,158 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01246ab14f3b4349ad36c53e0634824e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811287183 2024-12-10T06:14:48,167 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:48,168 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/b1fdefd2392b4b58ac5ecce004757a99 is 50, key is test_row_0/C:col10/1733811287183/Put/seqid=0 2024-12-10T06:14:48,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741978_1154 (size=12459) 2024-12-10T06:14:48,188 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/b1fdefd2392b4b58ac5ecce004757a99 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b1fdefd2392b4b58ac5ecce004757a99 2024-12-10T06:14:48,196 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/C of 9a8a17594d375f527e3c6e0f2fcae42e into b1fdefd2392b4b58ac5ecce004757a99(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:48,196 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:48,196 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/C, priority=13, startTime=1733811288089; duration=0sec 2024-12-10T06:14:48,196 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:48,196 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:C 2024-12-10T06:14:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:48,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-10T06:14:48,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:48,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:48,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:48,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:48,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:48,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:48,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210edfbfd0021054358a5702be16d52a0b7_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811287272/Put/seqid=0 2024-12-10T06:14:48,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741979_1155 (size=12304) 2024-12-10T06:14:48,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811348402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811348405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811348415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811348425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:48,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811348425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811348515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:48,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811348515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,530 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/aaf15bbc329a40be9740c49ba55c52de as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/aaf15bbc329a40be9740c49ba55c52de 2024-12-10T06:14:48,538 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/B of 9a8a17594d375f527e3c6e0f2fcae42e into aaf15bbc329a40be9740c49ba55c52de(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:48,538 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:48,538 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/B, priority=13, startTime=1733811288089; duration=0sec 2024-12-10T06:14:48,538 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:48,538 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:B 2024-12-10T06:14:48,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811348725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:48,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811348725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:48,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:48,805 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210edfbfd0021054358a5702be16d52a0b7_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210edfbfd0021054358a5702be16d52a0b7_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:48,806 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/6deb6e78facf49ceb38d81b2986f5891, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:48,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/6deb6e78facf49ceb38d81b2986f5891 is 175, key is test_row_0/A:col10/1733811287272/Put/seqid=0 2024-12-10T06:14:48,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741980_1156 (size=31105) 2024-12-10T06:14:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T06:14:48,975 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:48,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-10T06:14:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T06:14:48,978 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:48,979 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:48,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:49,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811349046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:49,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811349046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T06:14:49,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T06:14:49,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:49,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,211 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/6deb6e78facf49ceb38d81b2986f5891 2024-12-10T06:14:49,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/0808f1fa132d4f6b8b48718aa294b62d is 50, key is test_row_0/B:col10/1733811287272/Put/seqid=0 2024-12-10T06:14:49,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741981_1157 (size=12151) 2024-12-10T06:14:49,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T06:14:49,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T06:14:49,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T06:14:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811349435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:49,444 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 9a8a17594d375f527e3c6e0f2fcae42e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:49,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811349446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:49,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811349446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:49,449 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 9a8a17594d375f527e3c6e0f2fcae42e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:49,450 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 9a8a17594d375f527e3c6e0f2fcae42e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811349555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:49,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811349555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:49,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T06:14:49,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:49,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T06:14:49,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/0808f1fa132d4f6b8b48718aa294b62d 2024-12-10T06:14:49,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/d1e76290deed4e8f952c94faf53ad2e4 is 50, key is test_row_0/C:col10/1733811287272/Put/seqid=0 2024-12-10T06:14:49,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741982_1158 (size=12151) 2024-12-10T06:14:49,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T06:14:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T06:14:49,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:49,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:49,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:50,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/d1e76290deed4e8f952c94faf53ad2e4 2024-12-10T06:14:50,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/6deb6e78facf49ceb38d81b2986f5891 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/6deb6e78facf49ceb38d81b2986f5891 2024-12-10T06:14:50,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T06:14:50,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:50,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:50,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:50,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:50,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:50,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/6deb6e78facf49ceb38d81b2986f5891, entries=150, sequenceid=180, filesize=30.4 K 2024-12-10T06:14:50,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/0808f1fa132d4f6b8b48718aa294b62d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0808f1fa132d4f6b8b48718aa294b62d 2024-12-10T06:14:50,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0808f1fa132d4f6b8b48718aa294b62d, entries=150, sequenceid=180, filesize=11.9 K 2024-12-10T06:14:50,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/d1e76290deed4e8f952c94faf53ad2e4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/d1e76290deed4e8f952c94faf53ad2e4 2024-12-10T06:14:50,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/d1e76290deed4e8f952c94faf53ad2e4, entries=150, sequenceid=180, filesize=11.9 K 2024-12-10T06:14:50,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=40.25 KB/41220 for 9a8a17594d375f527e3c6e0f2fcae42e in 1696ms, sequenceid=180, compaction requested=false 2024-12-10T06:14:50,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T06:14:50,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T06:14:50,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:50,209 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T06:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103788d4699ff64a528dea4aee11ed819f_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811288388/Put/seqid=0 2024-12-10T06:14:50,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741983_1159 (size=12304) 2024-12-10T06:14:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:50,263 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103788d4699ff64a528dea4aee11ed819f_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103788d4699ff64a528dea4aee11ed819f_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:50,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/b76fa1283c9e4d2c9dcb363d86826c16, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/b76fa1283c9e4d2c9dcb363d86826c16 is 175, key is test_row_0/A:col10/1733811288388/Put/seqid=0 2024-12-10T06:14:50,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741984_1160 (size=31105) 2024-12-10T06:14:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:50,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:50,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811290622, max timestamp=1733811290638Num uniques 859; 2024-12-10T06:14:50,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:14:50,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811290622, max timestamp=1733811290638Num uniques 859; 2024-12-10T06:14:50,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:14:50,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811290622, max timestamp=1733811290638Num uniques 859; 2024-12-10T06:14:50,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:14:50,699 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/b76fa1283c9e4d2c9dcb363d86826c16 2024-12-10T06:14:50,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/ce84652a772748bda6b6643f215f7b79 is 50, key is test_row_0/B:col10/1733811288388/Put/seqid=0 2024-12-10T06:14:50,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811350767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:50,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811350770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:50,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741985_1161 (size=12151) 2024-12-10T06:14:50,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/ce84652a772748bda6b6643f215f7b79 2024-12-10T06:14:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/32806055834045878b86d0c6d38387c1 is 50, key is test_row_0/C:col10/1733811288388/Put/seqid=0 2024-12-10T06:14:50,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741986_1162 (size=12151) 2024-12-10T06:14:50,839 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/32806055834045878b86d0c6d38387c1 2024-12-10T06:14:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/b76fa1283c9e4d2c9dcb363d86826c16 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b76fa1283c9e4d2c9dcb363d86826c16 2024-12-10T06:14:50,858 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b76fa1283c9e4d2c9dcb363d86826c16, entries=150, sequenceid=190, filesize=30.4 K 2024-12-10T06:14:50,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/ce84652a772748bda6b6643f215f7b79 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ce84652a772748bda6b6643f215f7b79 2024-12-10T06:14:50,865 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ce84652a772748bda6b6643f215f7b79, entries=150, sequenceid=190, filesize=11.9 K 2024-12-10T06:14:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/32806055834045878b86d0c6d38387c1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/32806055834045878b86d0c6d38387c1 2024-12-10T06:14:50,871 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/32806055834045878b86d0c6d38387c1, entries=150, sequenceid=190, filesize=11.9 K 2024-12-10T06:14:50,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 9a8a17594d375f527e3c6e0f2fcae42e in 664ms, sequenceid=190, compaction requested=true 2024-12-10T06:14:50,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:50,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:50,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-10T06:14:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-10T06:14:50,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-10T06:14:50,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-12-10T06:14:50,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9100 sec 2024-12-10T06:14:50,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811290728, max timestamp=1733811290892Num uniques 1068; 2024-12-10T06:14:50,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T06:14:50,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811290728, max timestamp=1733811290892Num uniques 1068; 2024-12-10T06:14:50,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T06:14:50,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=67.09 KB heapSize=183.88 KB 2024-12-10T06:14:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:50,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811290728, max timestamp=1733811290900Num uniques 1068; 2024-12-10T06:14:50,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T06:14:50,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:50,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:50,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:50,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:50,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:50,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:50,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c5e6e3a614a047e2a746aeb58fa47703_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811290728/Put/seqid=0 2024-12-10T06:14:50,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741987_1163 (size=17284) 2024-12-10T06:14:50,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:50,987 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c5e6e3a614a047e2a746aeb58fa47703_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c5e6e3a614a047e2a746aeb58fa47703_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:50,988 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/567f51bad1464eb5ab2937c801b248e2, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:50,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/567f51bad1464eb5ab2937c801b248e2 is 175, key is test_row_0/A:col10/1733811290728/Put/seqid=0 2024-12-10T06:14:51,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811290976, max timestamp=1733811290998Num uniques 859; 2024-12-10T06:14:51,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:14:51,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811290976, max timestamp=1733811290999Num uniques 859; 2024-12-10T06:14:51,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:14:51,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811290976, max timestamp=1733811290999Num uniques 859; 2024-12-10T06:14:51,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:14:51,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741988_1164 (size=48389) 2024-12-10T06:14:51,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/567f51bad1464eb5ab2937c801b248e2 2024-12-10T06:14:51,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/8064da6de08e4d07b32d91fb81337fdc is 50, key is test_row_0/B:col10/1733811290728/Put/seqid=0 2024-12-10T06:14:51,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741989_1165 (size=12151) 2024-12-10T06:14:51,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811291090, max timestamp=1733811291115Num uniques 1018; 2024-12-10T06:14:51,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T06:14:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T06:14:51,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811291090, max timestamp=1733811291115Num uniques 1018; 2024-12-10T06:14:51,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T06:14:51,126 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:51,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:51,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811291090, max timestamp=1733811291115Num uniques 1018; 2024-12-10T06:14:51,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T06:14:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-10T06:14:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T06:14:51,130 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:51,130 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:51,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:51,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733811291198, max timestamp=1733811291208Num uniques 1027; 2024-12-10T06:14:51,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-10T06:14:51,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733811291198, max timestamp=1733811291208Num uniques 1027; 2024-12-10T06:14:51,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-10T06:14:51,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733811291198, max timestamp=1733811291208Num uniques 1027; 2024-12-10T06:14:51,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-10T06:14:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T06:14:51,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733811291264, max timestamp=1733811291281Num uniques 1086; 2024-12-10T06:14:51,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-10T06:14:51,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733811291264, max timestamp=1733811291281Num uniques 1086; 2024-12-10T06:14:51,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733811291264, max timestamp=1733811291281Num uniques 1086; 2024-12-10T06:14:51,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-10T06:14:51,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-10T06:14:51,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:51,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:51,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1733811291273, max timestamp=1733811291346Num uniques 1095; 2024-12-10T06:14:51,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=8920, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-10T06:14:51,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1733811291273, max timestamp=1733811291346Num uniques 1095; 2024-12-10T06:14:51,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=8920, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-10T06:14:51,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1733811291273, max timestamp=1733811291346Num uniques 1095; 2024-12-10T06:14:51,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=8920, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-10T06:14:51,407 INFO [master/c42c682b34f7:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-10T06:14:51,407 INFO [master/c42c682b34f7:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-10T06:14:51,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:51,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:51,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T06:14:51,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/8064da6de08e4d07b32d91fb81337fdc 2024-12-10T06:14:51,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811351457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733811291421, max timestamp=1733811291458Num uniques 1054; 2024-12-10T06:14:51,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-10T06:14:51,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733811291421, max timestamp=1733811291458Num uniques 1054; 2024-12-10T06:14:51,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733811291421, max timestamp=1733811291458Num uniques 1054; 2024-12-10T06:14:51,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-10T06:14:51,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-10T06:14:51,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/b749167b0b6e4592a270ef42c287954f is 50, key is test_row_0/C:col10/1733811290728/Put/seqid=0 2024-12-10T06:14:51,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741990_1166 (size=12151) 2024-12-10T06:14:51,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/b749167b0b6e4592a270ef42c287954f 2024-12-10T06:14:51,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733811291488, max timestamp=1733811291517Num uniques 1063; 2024-12-10T06:14:51,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-10T06:14:51,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733811291488, max timestamp=1733811291517Num uniques 1063; 2024-12-10T06:14:51,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-10T06:14:51,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733811291488, max timestamp=1733811291517Num uniques 1063; 2024-12-10T06:14:51,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-10T06:14:51,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/567f51bad1464eb5ab2937c801b248e2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/567f51bad1464eb5ab2937c801b248e2 2024-12-10T06:14:51,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/567f51bad1464eb5ab2937c801b248e2, entries=250, sequenceid=229, filesize=47.3 K 2024-12-10T06:14:51,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/8064da6de08e4d07b32d91fb81337fdc as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8064da6de08e4d07b32d91fb81337fdc 2024-12-10T06:14:51,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8064da6de08e4d07b32d91fb81337fdc, entries=150, sequenceid=229, filesize=11.9 K 2024-12-10T06:14:51,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/b749167b0b6e4592a270ef42c287954f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b749167b0b6e4592a270ef42c287954f 2024-12-10T06:14:51,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b749167b0b6e4592a270ef42c287954f, entries=150, sequenceid=229, filesize=11.9 K 2024-12-10T06:14:51,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=73.80 KB/75570 for 9a8a17594d375f527e3c6e0f2fcae42e in 653ms, sequenceid=229, compaction requested=true 2024-12-10T06:14:51,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:51,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:51,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:51,559 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:51,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:51,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:51,559 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:51,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:51,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:51,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=80.51 KB heapSize=220.55 KB 2024-12-10T06:14:51,561 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:51,561 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142012 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:51,561 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/B is initiating minor compaction (all files) 2024-12-10T06:14:51,561 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/A is initiating minor compaction (all files) 2024-12-10T06:14:51,561 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/B in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,561 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/A in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:51,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:51,561 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/aaf15bbc329a40be9740c49ba55c52de, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0808f1fa132d4f6b8b48718aa294b62d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ce84652a772748bda6b6643f215f7b79, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8064da6de08e4d07b32d91fb81337fdc] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=47.8 K 2024-12-10T06:14:51,561 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/88d6bb788555412da75207c8b83b6ca3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/6deb6e78facf49ceb38d81b2986f5891, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b76fa1283c9e4d2c9dcb363d86826c16, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/567f51bad1464eb5ab2937c801b248e2] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=138.7 K 2024-12-10T06:14:51,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:51,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:51,562 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:51,562 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/88d6bb788555412da75207c8b83b6ca3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/6deb6e78facf49ceb38d81b2986f5891, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b76fa1283c9e4d2c9dcb363d86826c16, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/567f51bad1464eb5ab2937c801b248e2] 2024-12-10T06:14:51,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:51,562 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d6bb788555412da75207c8b83b6ca3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811287183 2024-12-10T06:14:51,562 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting aaf15bbc329a40be9740c49ba55c52de, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811287183 2024-12-10T06:14:51,563 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 0808f1fa132d4f6b8b48718aa294b62d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733811287272 2024-12-10T06:14:51,563 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6deb6e78facf49ceb38d81b2986f5891, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733811287272 2024-12-10T06:14:51,563 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting b76fa1283c9e4d2c9dcb363d86826c16, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811288388 2024-12-10T06:14:51,564 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567f51bad1464eb5ab2937c801b248e2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733811290728 2024-12-10T06:14:51,564 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ce84652a772748bda6b6643f215f7b79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811288388 2024-12-10T06:14:51,565 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8064da6de08e4d07b32d91fb81337fdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733811290728 2024-12-10T06:14:51,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,600 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:51,601 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/09fffc6cc61d4483bd32911a9884767d is 50, key is test_row_0/B:col10/1733811290728/Put/seqid=0 2024-12-10T06:14:51,607 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:51,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a0ffae6bfac142a799c233bccf556d70_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811291540/Put/seqid=0 2024-12-10T06:14:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811351595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,622 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121091358d38a2714d9e99d256a9887c935e_9a8a17594d375f527e3c6e0f2fcae42e store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:51,625 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121091358d38a2714d9e99d256a9887c935e_9a8a17594d375f527e3c6e0f2fcae42e, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:51,626 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121091358d38a2714d9e99d256a9887c935e_9a8a17594d375f527e3c6e0f2fcae42e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:51,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811351613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811351615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811351618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811351618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741991_1167 (size=12595) 2024-12-10T06:14:51,677 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/09fffc6cc61d4483bd32911a9884767d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/09fffc6cc61d4483bd32911a9884767d 2024-12-10T06:14:51,684 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/B of 9a8a17594d375f527e3c6e0f2fcae42e into 09fffc6cc61d4483bd32911a9884767d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:51,684 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:51,684 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/B, priority=12, startTime=1733811291559; duration=0sec 2024-12-10T06:14:51,684 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:51,684 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:B 2024-12-10T06:14:51,684 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:14:51,687 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:14:51,687 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/C is initiating minor compaction (all files) 2024-12-10T06:14:51,687 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/C in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,687 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b1fdefd2392b4b58ac5ecce004757a99, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/d1e76290deed4e8f952c94faf53ad2e4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/32806055834045878b86d0c6d38387c1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b749167b0b6e4592a270ef42c287954f] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=47.8 K 2024-12-10T06:14:51,688 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b1fdefd2392b4b58ac5ecce004757a99, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811287183 2024-12-10T06:14:51,688 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e76290deed4e8f952c94faf53ad2e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733811287272 2024-12-10T06:14:51,689 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 32806055834045878b86d0c6d38387c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811288388 2024-12-10T06:14:51,689 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b749167b0b6e4592a270ef42c287954f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733811290728 2024-12-10T06:14:51,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741992_1168 (size=22618) 2024-12-10T06:14:51,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:51,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741993_1169 (size=4469) 2024-12-10T06:14:51,711 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#A#compaction#139 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:51,712 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/c4eceebbc2c94cee8b2b2540730a2856 is 175, key is test_row_0/A:col10/1733811290728/Put/seqid=0 2024-12-10T06:14:51,712 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a0ffae6bfac142a799c233bccf556d70_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a0ffae6bfac142a799c233bccf556d70_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:51,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/5ef9b85346454857a1b20faa4aa5aa15, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:51,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/5ef9b85346454857a1b20faa4aa5aa15 is 175, key is test_row_0/A:col10/1733811291540/Put/seqid=0 2024-12-10T06:14:51,728 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:51,728 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/0d60af3961ab45df8d9660397da0def6 is 50, key is test_row_0/C:col10/1733811290728/Put/seqid=0 2024-12-10T06:14:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811351725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:51,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:51,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811351736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811351736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T06:14:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811351736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811351736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741994_1170 (size=31549) 2024-12-10T06:14:51,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741995_1171 (size=66023) 2024-12-10T06:14:51,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741996_1172 (size=12595) 2024-12-10T06:14:51,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:51,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:51,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811351935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811351956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811351956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811351965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:51,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811351965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,185 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/c4eceebbc2c94cee8b2b2540730a2856 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/c4eceebbc2c94cee8b2b2540730a2856 2024-12-10T06:14:52,195 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/A of 9a8a17594d375f527e3c6e0f2fcae42e into c4eceebbc2c94cee8b2b2540730a2856(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:52,195 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:52,196 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/A, priority=12, startTime=1733811291559; duration=0sec 2024-12-10T06:14:52,196 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:52,196 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:A 2024-12-10T06:14:52,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/5ef9b85346454857a1b20faa4aa5aa15 2024-12-10T06:14:52,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:52,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,229 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/0d60af3961ab45df8d9660397da0def6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/0d60af3961ab45df8d9660397da0def6 2024-12-10T06:14:52,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/da8ea34f71a2441fbabe68a0260929d9 is 50, key is test_row_0/B:col10/1733811291540/Put/seqid=0 2024-12-10T06:14:52,239 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/C of 9a8a17594d375f527e3c6e0f2fcae42e into 0d60af3961ab45df8d9660397da0def6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:52,239 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:52,239 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/C, priority=12, startTime=1733811291559; duration=0sec 2024-12-10T06:14:52,239 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:52,239 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:C 2024-12-10T06:14:52,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811352246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T06:14:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811352267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741997_1173 (size=12301) 2024-12-10T06:14:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811352267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811352275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811352275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/da8ea34f71a2441fbabe68a0260929d9 2024-12-10T06:14:52,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5874e1f75be0401cb9b35437edbcc014 is 50, key is test_row_0/C:col10/1733811291540/Put/seqid=0 2024-12-10T06:14:52,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741998_1174 (size=12301) 2024-12-10T06:14:52,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5874e1f75be0401cb9b35437edbcc014 2024-12-10T06:14:52,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/5ef9b85346454857a1b20faa4aa5aa15 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5ef9b85346454857a1b20faa4aa5aa15 2024-12-10T06:14:52,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5ef9b85346454857a1b20faa4aa5aa15, entries=350, sequenceid=360, filesize=64.5 K 2024-12-10T06:14:52,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/da8ea34f71a2441fbabe68a0260929d9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/da8ea34f71a2441fbabe68a0260929d9 2024-12-10T06:14:52,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/da8ea34f71a2441fbabe68a0260929d9, entries=150, sequenceid=360, filesize=12.0 K 2024-12-10T06:14:52,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5874e1f75be0401cb9b35437edbcc014 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5874e1f75be0401cb9b35437edbcc014 2024-12-10T06:14:52,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5874e1f75be0401cb9b35437edbcc014, entries=150, sequenceid=360, filesize=12.0 K 2024-12-10T06:14:52,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~220.50 KB/225792, currentSize=107.34 KB/109920 for 9a8a17594d375f527e3c6e0f2fcae42e in 1174ms, sequenceid=360, compaction requested=false 2024-12-10T06:14:52,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:52,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T06:14:52,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:52,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:52,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:52,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:52,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:52,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:52,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108c566113cc334e1090aab6c72512559b_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811291614/Put/seqid=0 2024-12-10T06:14:52,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741999_1175 (size=14994) 2024-12-10T06:14:52,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811352784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811352786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811352787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811352787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811352789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811352895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811352895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811352895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:52,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:52,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:52,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:52,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811353105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811353105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811353108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:53,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:53,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:53,188 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108c566113cc334e1090aab6c72512559b_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108c566113cc334e1090aab6c72512559b_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:53,189 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/0bf20b972fd04784a755520c71eafa57, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:53,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/0bf20b972fd04784a755520c71eafa57 is 175, key is test_row_0/A:col10/1733811291614/Put/seqid=0 2024-12-10T06:14:53,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742000_1176 (size=39949) 2024-12-10T06:14:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T06:14:53,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811353415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811353415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811353415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:53,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:53,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/0bf20b972fd04784a755520c71eafa57 2024-12-10T06:14:53,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:53,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/740f5f0e4e6d4d0a81149f07cf698005 is 50, key is test_row_0/B:col10/1733811291614/Put/seqid=0 2024-12-10T06:14:53,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742001_1177 (size=12301) 2024-12-10T06:14:53,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/740f5f0e4e6d4d0a81149f07cf698005 2024-12-10T06:14:53,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/f2f21c644e2146f68befc2896ec77631 is 50, key is test_row_0/C:col10/1733811291614/Put/seqid=0 2024-12-10T06:14:53,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742002_1178 (size=12301) 2024-12-10T06:14:53,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811353805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811353805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,809 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 9a8a17594d375f527e3c6e0f2fcae42e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,810 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 9a8a17594d375f527e3c6e0f2fcae42e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:53,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:53,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811353925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811353925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811353925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:54,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:54,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:54,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:54,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:54,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:54,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:54,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/f2f21c644e2146f68befc2896ec77631 2024-12-10T06:14:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/0bf20b972fd04784a755520c71eafa57 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0bf20b972fd04784a755520c71eafa57 2024-12-10T06:14:54,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0bf20b972fd04784a755520c71eafa57, entries=200, sequenceid=383, filesize=39.0 K 2024-12-10T06:14:54,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/740f5f0e4e6d4d0a81149f07cf698005 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/740f5f0e4e6d4d0a81149f07cf698005 2024-12-10T06:14:54,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/740f5f0e4e6d4d0a81149f07cf698005, entries=150, sequenceid=383, filesize=12.0 K 2024-12-10T06:14:54,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/f2f21c644e2146f68befc2896ec77631 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f2f21c644e2146f68befc2896ec77631 2024-12-10T06:14:54,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f2f21c644e2146f68befc2896ec77631, entries=150, sequenceid=383, filesize=12.0 K 2024-12-10T06:14:54,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9a8a17594d375f527e3c6e0f2fcae42e in 1380ms, sequenceid=383, compaction requested=true 2024-12-10T06:14:54,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:54,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:54,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:54,138 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:54,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:54,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:54,138 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:54,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:54,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:54,139 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:54,139 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/A is initiating minor compaction (all files) 2024-12-10T06:14:54,139 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/A in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:54,139 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/c4eceebbc2c94cee8b2b2540730a2856, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5ef9b85346454857a1b20faa4aa5aa15, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0bf20b972fd04784a755520c71eafa57] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=134.3 K 2024-12-10T06:14:54,139 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:54,139 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/c4eceebbc2c94cee8b2b2540730a2856, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5ef9b85346454857a1b20faa4aa5aa15, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0bf20b972fd04784a755520c71eafa57] 2024-12-10T06:14:54,140 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:54,140 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4eceebbc2c94cee8b2b2540730a2856, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733811290728 2024-12-10T06:14:54,140 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/B is initiating minor compaction (all files) 2024-12-10T06:14:54,140 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/B in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:54,140 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/09fffc6cc61d4483bd32911a9884767d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/da8ea34f71a2441fbabe68a0260929d9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/740f5f0e4e6d4d0a81149f07cf698005] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.3 K 2024-12-10T06:14:54,141 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef9b85346454857a1b20faa4aa5aa15, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733811291512 2024-12-10T06:14:54,141 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fffc6cc61d4483bd32911a9884767d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733811290728 2024-12-10T06:14:54,141 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bf20b972fd04784a755520c71eafa57, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733811291610 2024-12-10T06:14:54,141 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting da8ea34f71a2441fbabe68a0260929d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733811291540 2024-12-10T06:14:54,142 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 740f5f0e4e6d4d0a81149f07cf698005, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733811291612 2024-12-10T06:14:54,150 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:54,152 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:54,153 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/7ec47e4f2c074feb93e9e277e68b2e45 is 50, key is test_row_0/B:col10/1733811291614/Put/seqid=0 2024-12-10T06:14:54,154 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210fc9dc65901fb495cb3eaca2bbfc1383f_9a8a17594d375f527e3c6e0f2fcae42e store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:54,157 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210fc9dc65901fb495cb3eaca2bbfc1383f_9a8a17594d375f527e3c6e0f2fcae42e, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:54,157 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fc9dc65901fb495cb3eaca2bbfc1383f_9a8a17594d375f527e3c6e0f2fcae42e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:54,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742003_1179 (size=12847) 2024-12-10T06:14:54,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742004_1180 (size=4469) 2024-12-10T06:14:54,172 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#A#compaction#147 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:54,173 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ad9e560b316e4e4dbe3d33fbdefeffde is 175, key is test_row_0/A:col10/1733811291614/Put/seqid=0 2024-12-10T06:14:54,178 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/7ec47e4f2c074feb93e9e277e68b2e45 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/7ec47e4f2c074feb93e9e277e68b2e45 2024-12-10T06:14:54,186 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/B of 9a8a17594d375f527e3c6e0f2fcae42e into 7ec47e4f2c074feb93e9e277e68b2e45(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:54,186 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:54,186 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/B, priority=13, startTime=1733811294138; duration=0sec 2024-12-10T06:14:54,186 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:54,186 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:B 2024-12-10T06:14:54,186 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:54,187 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:54,187 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/C is initiating minor compaction (all files) 2024-12-10T06:14:54,187 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/C in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:54,187 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/0d60af3961ab45df8d9660397da0def6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5874e1f75be0401cb9b35437edbcc014, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f2f21c644e2146f68befc2896ec77631] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.3 K 2024-12-10T06:14:54,188 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d60af3961ab45df8d9660397da0def6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733811290728 2024-12-10T06:14:54,188 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 5874e1f75be0401cb9b35437edbcc014, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733811291540 2024-12-10T06:14:54,189 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f21c644e2146f68befc2896ec77631, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733811291612 2024-12-10T06:14:54,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742005_1181 (size=31801) 2024-12-10T06:14:54,204 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:54,205 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/7d5e5959a49d45fa9551412fc554c78a is 50, key is test_row_0/C:col10/1733811291614/Put/seqid=0 2024-12-10T06:14:54,208 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/ad9e560b316e4e4dbe3d33fbdefeffde as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ad9e560b316e4e4dbe3d33fbdefeffde 2024-12-10T06:14:54,218 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/A of 9a8a17594d375f527e3c6e0f2fcae42e into ad9e560b316e4e4dbe3d33fbdefeffde(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:54,218 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:54,218 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/A, priority=13, startTime=1733811294137; duration=0sec 2024-12-10T06:14:54,218 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:54,218 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:A 2024-12-10T06:14:54,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T06:14:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:54,220 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T06:14:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:54,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742006_1182 (size=12847) 2024-12-10T06:14:54,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cccbb1fd33a0486e9fa89bc9cade9112_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811292782/Put/seqid=0 2024-12-10T06:14:54,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742007_1183 (size=12454) 2024-12-10T06:14:54,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,283 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cccbb1fd33a0486e9fa89bc9cade9112_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cccbb1fd33a0486e9fa89bc9cade9112_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/0e9b9afc83b444959fd97bf2f123ba13, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:54,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/0e9b9afc83b444959fd97bf2f123ba13 is 175, key is test_row_0/A:col10/1733811292782/Put/seqid=0 2024-12-10T06:14:54,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742008_1184 (size=31255) 2024-12-10T06:14:54,301 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/0e9b9afc83b444959fd97bf2f123ba13 2024-12-10T06:14:54,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/e2f5248b9b3148aabcbc5be6695e2a7d is 50, key is test_row_0/B:col10/1733811292782/Put/seqid=0 2024-12-10T06:14:54,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742009_1185 (size=12301) 2024-12-10T06:14:54,331 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/e2f5248b9b3148aabcbc5be6695e2a7d 2024-12-10T06:14:54,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/f24a35a831824373ba4a1cf900ca533b is 50, key is test_row_0/C:col10/1733811292782/Put/seqid=0 2024-12-10T06:14:54,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742010_1186 (size=12301) 2024-12-10T06:14:54,660 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/7d5e5959a49d45fa9551412fc554c78a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7d5e5959a49d45fa9551412fc554c78a 2024-12-10T06:14:54,668 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/C of 9a8a17594d375f527e3c6e0f2fcae42e into 7d5e5959a49d45fa9551412fc554c78a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:54,668 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:54,668 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/C, priority=13, startTime=1733811294138; duration=0sec 2024-12-10T06:14:54,668 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:54,668 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:C 2024-12-10T06:14:54,760 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/f24a35a831824373ba4a1cf900ca533b 2024-12-10T06:14:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/0e9b9afc83b444959fd97bf2f123ba13 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0e9b9afc83b444959fd97bf2f123ba13 2024-12-10T06:14:54,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0e9b9afc83b444959fd97bf2f123ba13, entries=150, sequenceid=397, filesize=30.5 K 2024-12-10T06:14:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/e2f5248b9b3148aabcbc5be6695e2a7d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e2f5248b9b3148aabcbc5be6695e2a7d 2024-12-10T06:14:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,783 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e2f5248b9b3148aabcbc5be6695e2a7d, entries=150, sequenceid=397, filesize=12.0 K 2024-12-10T06:14:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/f24a35a831824373ba4a1cf900ca533b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f24a35a831824373ba4a1cf900ca533b 2024-12-10T06:14:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f24a35a831824373ba4a1cf900ca533b, entries=150, sequenceid=397, filesize=12.0 K 2024-12-10T06:14:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 9a8a17594d375f527e3c6e0f2fcae42e in 571ms, sequenceid=397, compaction requested=false 2024-12-10T06:14:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:54,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-10T06:14:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-10T06:14:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-10T06:14:54,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6630 sec 2024-12-10T06:14:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.6700 sec 2024-12-10T06:14:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:54,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:14:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105a804a12913141bcb3a13916d92a0901_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811294989/Put/seqid=0 2024-12-10T06:14:55,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742011_1187 (size=22618) 2024-12-10T06:14:55,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811295042, max timestamp=1733811295053Num uniques 859; 2024-12-10T06:14:55,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811295042, max timestamp=1733811295053Num uniques 859; 2024-12-10T06:14:55,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:14:55,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:14:55,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811295042, max timestamp=1733811295053Num uniques 859; 2024-12-10T06:14:55,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:14:55,061 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:55,066 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105a804a12913141bcb3a13916d92a0901_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105a804a12913141bcb3a13916d92a0901_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:55,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/10e89bb23ff74c5492d5cc99be44ec24, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:55,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/10e89bb23ff74c5492d5cc99be44ec24 is 175, key is test_row_0/A:col10/1733811294989/Put/seqid=0 2024-12-10T06:14:55,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742012_1188 (size=66023) 2024-12-10T06:14:55,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811355107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811355107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811355117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811355225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811355229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811355229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T06:14:55,306 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:55,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-10T06:14:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T06:14:55,310 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:55,311 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:55,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T06:14:55,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811355445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811355446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811355446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T06:14:55,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:55,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:55,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:55,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,503 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/10e89bb23ff74c5492d5cc99be44ec24 2024-12-10T06:14:55,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/f1fb5a85988c4c2faeb7b010ff7a339c is 50, key is test_row_0/B:col10/1733811294989/Put/seqid=0 2024-12-10T06:14:55,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742013_1189 (size=12301) 2024-12-10T06:14:55,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/f1fb5a85988c4c2faeb7b010ff7a339c 2024-12-10T06:14:55,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5deee539a9cb42a1ab88312c129c665a is 50, key is test_row_0/C:col10/1733811294989/Put/seqid=0 2024-12-10T06:14:55,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742014_1190 (size=12301) 2024-12-10T06:14:55,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T06:14:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:55,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T06:14:55,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811355756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811355756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811355766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T06:14:55,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:55,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:55,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:55,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811355815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,820 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 9a8a17594d375f527e3c6e0f2fcae42e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811355825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:55,831 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 9a8a17594d375f527e3c6e0f2fcae42e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:14:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T06:14:55,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T06:14:55,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:55,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:55,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:55,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5deee539a9cb42a1ab88312c129c665a 2024-12-10T06:14:56,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/10e89bb23ff74c5492d5cc99be44ec24 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/10e89bb23ff74c5492d5cc99be44ec24 2024-12-10T06:14:56,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/10e89bb23ff74c5492d5cc99be44ec24, entries=350, sequenceid=408, filesize=64.5 K 2024-12-10T06:14:56,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/f1fb5a85988c4c2faeb7b010ff7a339c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/f1fb5a85988c4c2faeb7b010ff7a339c 2024-12-10T06:14:56,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/f1fb5a85988c4c2faeb7b010ff7a339c, entries=150, sequenceid=408, filesize=12.0 K 2024-12-10T06:14:56,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5deee539a9cb42a1ab88312c129c665a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5deee539a9cb42a1ab88312c129c665a 2024-12-10T06:14:56,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5deee539a9cb42a1ab88312c129c665a, entries=150, sequenceid=408, filesize=12.0 K 2024-12-10T06:14:56,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9a8a17594d375f527e3c6e0f2fcae42e in 1049ms, sequenceid=408, compaction requested=true 2024-12-10T06:14:56,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:56,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:56,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:56,046 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:56,047 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:56,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:56,048 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:56,048 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/B is initiating minor compaction (all files) 2024-12-10T06:14:56,049 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/B in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,049 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/7ec47e4f2c074feb93e9e277e68b2e45, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e2f5248b9b3148aabcbc5be6695e2a7d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/f1fb5a85988c4c2faeb7b010ff7a339c] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.6 K 2024-12-10T06:14:56,049 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129079 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:56,049 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/A is initiating minor compaction (all files) 2024-12-10T06:14:56,049 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/A in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,049 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ad9e560b316e4e4dbe3d33fbdefeffde, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0e9b9afc83b444959fd97bf2f123ba13, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/10e89bb23ff74c5492d5cc99be44ec24] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=126.1 K 2024-12-10T06:14:56,049 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,049 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ad9e560b316e4e4dbe3d33fbdefeffde, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0e9b9afc83b444959fd97bf2f123ba13, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/10e89bb23ff74c5492d5cc99be44ec24] 2024-12-10T06:14:56,052 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec47e4f2c074feb93e9e277e68b2e45, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733811291612 2024-12-10T06:14:56,052 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9e560b316e4e4dbe3d33fbdefeffde, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733811291612 2024-12-10T06:14:56,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:56,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:56,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:56,065 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e9b9afc83b444959fd97bf2f123ba13, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733811292782 2024-12-10T06:14:56,065 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f5248b9b3148aabcbc5be6695e2a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733811292782 2024-12-10T06:14:56,066 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting f1fb5a85988c4c2faeb7b010ff7a339c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733811294989 2024-12-10T06:14:56,066 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10e89bb23ff74c5492d5cc99be44ec24, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733811294970 2024-12-10T06:14:56,081 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:56,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T06:14:56,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,091 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T06:14:56,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:56,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:56,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:56,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:56,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:56,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:14:56,095 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:56,096 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/2e3e6d2e4ff74dddb347d9c8d1d6f7b4 is 50, key is test_row_0/B:col10/1733811294989/Put/seqid=0 2024-12-10T06:14:56,101 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121024b4fae052e9401f900c619eff46e1c3_9a8a17594d375f527e3c6e0f2fcae42e store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:56,103 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121024b4fae052e9401f900c619eff46e1c3_9a8a17594d375f527e3c6e0f2fcae42e, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:56,104 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121024b4fae052e9401f900c619eff46e1c3_9a8a17594d375f527e3c6e0f2fcae42e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:56,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742015_1191 (size=12949) 2024-12-10T06:14:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121013536b2525d643cb8096a96fcf2be314_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811295114/Put/seqid=0 2024-12-10T06:14:56,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742016_1192 (size=4469) 2024-12-10T06:14:56,203 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#A#compaction#156 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:56,204 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/05a7b01ed8f74952a7d37d7e6b177799 is 175, key is test_row_0/A:col10/1733811294989/Put/seqid=0 2024-12-10T06:14:56,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742018_1194 (size=31903) 2024-12-10T06:14:56,229 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/05a7b01ed8f74952a7d37d7e6b177799 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/05a7b01ed8f74952a7d37d7e6b177799 2024-12-10T06:14:56,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742017_1193 (size=12454) 2024-12-10T06:14:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:56,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121013536b2525d643cb8096a96fcf2be314_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121013536b2525d643cb8096a96fcf2be314_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:56,244 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/A of 9a8a17594d375f527e3c6e0f2fcae42e into 05a7b01ed8f74952a7d37d7e6b177799(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:56,244 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:56,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/db144b06a4bc44c39a3dbdce2003d5ed, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:56,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/db144b06a4bc44c39a3dbdce2003d5ed is 175, key is test_row_0/A:col10/1733811295114/Put/seqid=0 2024-12-10T06:14:56,246 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/A, priority=13, startTime=1733811296046; duration=0sec 2024-12-10T06:14:56,246 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:56,246 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:A 2024-12-10T06:14:56,247 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:56,249 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:56,249 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/C is initiating minor compaction (all files) 2024-12-10T06:14:56,249 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/C in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,249 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7d5e5959a49d45fa9551412fc554c78a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f24a35a831824373ba4a1cf900ca533b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5deee539a9cb42a1ab88312c129c665a] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.6 K 2024-12-10T06:14:56,250 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5e5959a49d45fa9551412fc554c78a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733811291612 2024-12-10T06:14:56,250 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f24a35a831824373ba4a1cf900ca533b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733811292782 2024-12-10T06:14:56,251 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5deee539a9cb42a1ab88312c129c665a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733811294989 2024-12-10T06:14:56,271 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:56,272 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5a9aee124e7f4000abf90f849c8712d5 is 50, key is test_row_0/C:col10/1733811294989/Put/seqid=0 2024-12-10T06:14:56,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:56,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742019_1195 (size=31255) 2024-12-10T06:14:56,281 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=444, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/db144b06a4bc44c39a3dbdce2003d5ed 2024-12-10T06:14:56,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742020_1196 (size=12949) 2024-12-10T06:14:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/ac95dffd03b949479b8c1f580724f8ec is 50, key is test_row_0/B:col10/1733811295114/Put/seqid=0 2024-12-10T06:14:56,296 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5a9aee124e7f4000abf90f849c8712d5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5a9aee124e7f4000abf90f849c8712d5 2024-12-10T06:14:56,302 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/C of 9a8a17594d375f527e3c6e0f2fcae42e into 5a9aee124e7f4000abf90f849c8712d5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:56,302 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:56,302 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/C, priority=13, startTime=1733811296052; duration=0sec 2024-12-10T06:14:56,302 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:56,302 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:C 2024-12-10T06:14:56,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811356297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811356305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811356306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742021_1197 (size=12301) 2024-12-10T06:14:56,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/ac95dffd03b949479b8c1f580724f8ec 2024-12-10T06:14:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/7489faf972a74d27bb1755c85ce08b96 is 50, key is test_row_0/C:col10/1733811295114/Put/seqid=0 2024-12-10T06:14:56,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742022_1198 (size=12301) 2024-12-10T06:14:56,375 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/7489faf972a74d27bb1755c85ce08b96 2024-12-10T06:14:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/db144b06a4bc44c39a3dbdce2003d5ed as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db144b06a4bc44c39a3dbdce2003d5ed 2024-12-10T06:14:56,387 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db144b06a4bc44c39a3dbdce2003d5ed, entries=150, sequenceid=444, filesize=30.5 K 2024-12-10T06:14:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/ac95dffd03b949479b8c1f580724f8ec as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ac95dffd03b949479b8c1f580724f8ec 2024-12-10T06:14:56,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ac95dffd03b949479b8c1f580724f8ec, entries=150, sequenceid=444, filesize=12.0 K 2024-12-10T06:14:56,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/7489faf972a74d27bb1755c85ce08b96 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7489faf972a74d27bb1755c85ce08b96 2024-12-10T06:14:56,416 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7489faf972a74d27bb1755c85ce08b96, entries=150, sequenceid=444, filesize=12.0 K 2024-12-10T06:14:56,418 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 9a8a17594d375f527e3c6e0f2fcae42e in 326ms, sequenceid=444, compaction requested=false 2024-12-10T06:14:56,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:56,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-10T06:14:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-10T06:14:56,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-10T06:14:56,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1080 sec 2024-12-10T06:14:56,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.1170 sec 2024-12-10T06:14:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:56,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T06:14:56,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:56,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:56,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:56,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:56,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:56,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T06:14:56,446 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:14:56,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:14:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-10T06:14:56,449 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:14:56,450 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:14:56,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:14:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T06:14:56,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d249d0ce044846159c8ce0a1f0830929_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811296429/Put/seqid=0 2024-12-10T06:14:56,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742023_1199 (size=17534) 2024-12-10T06:14:56,474 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:56,479 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d249d0ce044846159c8ce0a1f0830929_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d249d0ce044846159c8ce0a1f0830929_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:56,480 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/032b57fdc7384cefa3a68e6016d7c0e6, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:56,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/032b57fdc7384cefa3a68e6016d7c0e6 is 175, key is test_row_0/A:col10/1733811296429/Put/seqid=0 2024-12-10T06:14:56,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742024_1200 (size=48639) 2024-12-10T06:14:56,499 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/032b57fdc7384cefa3a68e6016d7c0e6 2024-12-10T06:14:56,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811356504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811356507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/b66dcabee71140dba471e720a6c6638a is 50, key is test_row_0/B:col10/1733811296429/Put/seqid=0 2024-12-10T06:14:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811356511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T06:14:56,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742025_1201 (size=12301) 2024-12-10T06:14:56,594 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/2e3e6d2e4ff74dddb347d9c8d1d6f7b4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2e3e6d2e4ff74dddb347d9c8d1d6f7b4 2024-12-10T06:14:56,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T06:14:56,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:56,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,608 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/B of 9a8a17594d375f527e3c6e0f2fcae42e into 2e3e6d2e4ff74dddb347d9c8d1d6f7b4(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:56,608 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:56,608 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/B, priority=13, startTime=1733811296046; duration=0sec 2024-12-10T06:14:56,608 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:56,608 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:B 2024-12-10T06:14:56,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811356625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811356625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811356635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T06:14:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T06:14:56,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811356835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811356835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811356845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:56,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:56,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:56,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T06:14:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:56,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:56,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/b66dcabee71140dba471e720a6c6638a 2024-12-10T06:14:56,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/18dcbb92e9684007ab4e3441c4f84b67 is 50, key is test_row_0/C:col10/1733811296429/Put/seqid=0 2024-12-10T06:14:57,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742026_1202 (size=12301) 2024-12-10T06:14:57,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T06:14:57,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T06:14:57,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811357146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:57,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811357146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:57,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811357166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:57,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T06:14:57,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:57,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:57,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:57,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T06:14:57,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:57,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:14:57,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/18dcbb92e9684007ab4e3441c4f84b67 2024-12-10T06:14:57,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/032b57fdc7384cefa3a68e6016d7c0e6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/032b57fdc7384cefa3a68e6016d7c0e6 2024-12-10T06:14:57,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/032b57fdc7384cefa3a68e6016d7c0e6, entries=250, sequenceid=458, filesize=47.5 K 2024-12-10T06:14:57,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/b66dcabee71140dba471e720a6c6638a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/b66dcabee71140dba471e720a6c6638a 2024-12-10T06:14:57,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/b66dcabee71140dba471e720a6c6638a, entries=150, sequenceid=458, filesize=12.0 K 2024-12-10T06:14:57,442 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T06:14:57,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/18dcbb92e9684007ab4e3441c4f84b67 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/18dcbb92e9684007ab4e3441c4f84b67 2024-12-10T06:14:57,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/18dcbb92e9684007ab4e3441c4f84b67, entries=150, sequenceid=458, filesize=12.0 K 2024-12-10T06:14:57,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9a8a17594d375f527e3c6e0f2fcae42e in 1026ms, sequenceid=458, compaction requested=true 2024-12-10T06:14:57,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:14:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:14:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:14:57,457 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:57,457 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:57,472 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:57,473 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/A is initiating minor compaction (all files) 2024-12-10T06:14:57,473 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/A in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,473 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/05a7b01ed8f74952a7d37d7e6b177799, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db144b06a4bc44c39a3dbdce2003d5ed, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/032b57fdc7384cefa3a68e6016d7c0e6] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=109.2 K 2024-12-10T06:14:57,473 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,473 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/05a7b01ed8f74952a7d37d7e6b177799, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db144b06a4bc44c39a3dbdce2003d5ed, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/032b57fdc7384cefa3a68e6016d7c0e6] 2024-12-10T06:14:57,473 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:57,473 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/B is initiating minor compaction (all files) 2024-12-10T06:14:57,474 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/B in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,474 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2e3e6d2e4ff74dddb347d9c8d1d6f7b4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ac95dffd03b949479b8c1f580724f8ec, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/b66dcabee71140dba471e720a6c6638a] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.7 K 2024-12-10T06:14:57,474 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05a7b01ed8f74952a7d37d7e6b177799, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733811294989 2024-12-10T06:14:57,474 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3e6d2e4ff74dddb347d9c8d1d6f7b4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733811294989 2024-12-10T06:14:57,475 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting db144b06a4bc44c39a3dbdce2003d5ed, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733811295104 2024-12-10T06:14:57,475 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ac95dffd03b949479b8c1f580724f8ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733811295104 2024-12-10T06:14:57,476 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 032b57fdc7384cefa3a68e6016d7c0e6, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733811296292 2024-12-10T06:14:57,476 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b66dcabee71140dba471e720a6c6638a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733811296294 2024-12-10T06:14:57,489 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:57,497 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:57,498 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/2efe93e1228c485fb189c169cdaec615 is 50, key is test_row_0/B:col10/1733811296429/Put/seqid=0 2024-12-10T06:14:57,500 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210c53d987b7fcf4e81b2e7e07f87d9c9b6_9a8a17594d375f527e3c6e0f2fcae42e store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:57,502 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210c53d987b7fcf4e81b2e7e07f87d9c9b6_9a8a17594d375f527e3c6e0f2fcae42e, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:57,503 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c53d987b7fcf4e81b2e7e07f87d9c9b6_9a8a17594d375f527e3c6e0f2fcae42e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:57,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742027_1203 (size=13051) 2024-12-10T06:14:57,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T06:14:57,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,527 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T06:14:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:57,534 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/2efe93e1228c485fb189c169cdaec615 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2efe93e1228c485fb189c169cdaec615 2024-12-10T06:14:57,543 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/B of 9a8a17594d375f527e3c6e0f2fcae42e into 2efe93e1228c485fb189c169cdaec615(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:57,544 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:57,544 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/B, priority=13, startTime=1733811297457; duration=0sec 2024-12-10T06:14:57,544 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:14:57,544 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:B 2024-12-10T06:14:57,544 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:14:57,545 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:14:57,545 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/C is initiating minor compaction (all files) 2024-12-10T06:14:57,545 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/C in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:57,545 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5a9aee124e7f4000abf90f849c8712d5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7489faf972a74d27bb1755c85ce08b96, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/18dcbb92e9684007ab4e3441c4f84b67] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.7 K 2024-12-10T06:14:57,546 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a9aee124e7f4000abf90f849c8712d5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733811294989 2024-12-10T06:14:57,546 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7489faf972a74d27bb1755c85ce08b96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733811295104 2024-12-10T06:14:57,547 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 18dcbb92e9684007ab4e3441c4f84b67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733811296294 2024-12-10T06:14:57,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742028_1204 (size=4469) 2024-12-10T06:14:57,549 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#A#compaction#165 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:57,550 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/eac8d6b828b64040abec134437849d03 is 175, key is test_row_0/A:col10/1733811296429/Put/seqid=0 2024-12-10T06:14:57,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210081c3826acef47159173b39322a12701_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811296501/Put/seqid=0 2024-12-10T06:14:57,576 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#C#compaction#168 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:14:57,577 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/cd8f99eed88043e5b929aaf26aa6ac27 is 50, key is test_row_0/C:col10/1733811296429/Put/seqid=0 2024-12-10T06:14:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T06:14:57,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742029_1205 (size=32005) 2024-12-10T06:14:57,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742030_1206 (size=12454) 2024-12-10T06:14:57,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742031_1207 (size=13051) 2024-12-10T06:14:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:57,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:14:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811357690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811357690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:57,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811357692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811357805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811357805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811357805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:14:58,016 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/eac8d6b828b64040abec134437849d03 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/eac8d6b828b64040abec134437849d03 2024-12-10T06:14:58,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210081c3826acef47159173b39322a12701_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210081c3826acef47159173b39322a12701_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:58,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811358015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:58,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/989fddddcf2d4360907f641f88127bff, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:14:58,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811358015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811358016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:58,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/989fddddcf2d4360907f641f88127bff is 175, key is test_row_0/A:col10/1733811296501/Put/seqid=0 2024-12-10T06:14:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,026 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/A of 9a8a17594d375f527e3c6e0f2fcae42e into eac8d6b828b64040abec134437849d03(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:58,026 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:58,026 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/A, priority=13, startTime=1733811297457; duration=0sec 2024-12-10T06:14:58,026 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:58,026 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:A 2024-12-10T06:14:58,031 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/cd8f99eed88043e5b929aaf26aa6ac27 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/cd8f99eed88043e5b929aaf26aa6ac27 2024-12-10T06:14:58,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742032_1208 (size=31255) 2024-12-10T06:14:58,040 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/C of 9a8a17594d375f527e3c6e0f2fcae42e into cd8f99eed88043e5b929aaf26aa6ac27(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:14:58,040 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:58,040 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/C, priority=13, startTime=1733811297457; duration=0sec 2024-12-10T06:14:58,040 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:14:58,041 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:C 2024-12-10T06:14:58,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811358325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:58,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811358326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811358326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,438 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=479, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/989fddddcf2d4360907f641f88127bff 2024-12-10T06:14:58,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/73c001594efe412c80623588ad97b427 is 50, key is test_row_0/B:col10/1733811296501/Put/seqid=0 2024-12-10T06:14:58,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742033_1209 (size=12301) 2024-12-10T06:14:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T06:14:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811358836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811358836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811358836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:58,871 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/73c001594efe412c80623588ad97b427 2024-12-10T06:14:58,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/ec9ab93fe00a4eb9945704c953757064 is 50, key is test_row_0/C:col10/1733811296501/Put/seqid=0 2024-12-10T06:14:58,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742034_1210 (size=12301) 2024-12-10T06:14:59,299 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/ec9ab93fe00a4eb9945704c953757064 2024-12-10T06:14:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/989fddddcf2d4360907f641f88127bff as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/989fddddcf2d4360907f641f88127bff 2024-12-10T06:14:59,310 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/989fddddcf2d4360907f641f88127bff, entries=150, sequenceid=479, filesize=30.5 K 2024-12-10T06:14:59,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/73c001594efe412c80623588ad97b427 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/73c001594efe412c80623588ad97b427 2024-12-10T06:14:59,316 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/73c001594efe412c80623588ad97b427, entries=150, sequenceid=479, filesize=12.0 K 2024-12-10T06:14:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/ec9ab93fe00a4eb9945704c953757064 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ec9ab93fe00a4eb9945704c953757064 2024-12-10T06:14:59,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ec9ab93fe00a4eb9945704c953757064, entries=150, sequenceid=479, filesize=12.0 K 2024-12-10T06:14:59,325 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 9a8a17594d375f527e3c6e0f2fcae42e in 1797ms, sequenceid=479, compaction requested=false 2024-12-10T06:14:59,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:14:59,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:14:59,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-10T06:14:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-10T06:14:59,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-10T06:14:59,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8760 sec 2024-12-10T06:14:59,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.8830 sec 2024-12-10T06:14:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:14:59,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T06:14:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:14:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:14:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:14:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:14:59,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210375aed4b467d48d88b02056d9b37dd57_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811299838/Put/seqid=0 2024-12-10T06:14:59,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811359886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:59,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811359887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811359887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:59,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811359888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:59,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:14:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811359890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:14:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:14:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:14:59,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742035_1211 (size=12454) 2024-12-10T06:15:00,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811360005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811360005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811360005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811360006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811360015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811360215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811360215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811360215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811360215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811360225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,310 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:00,315 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210375aed4b467d48d88b02056d9b37dd57_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210375aed4b467d48d88b02056d9b37dd57_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:00,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/742e9da51daf4ba384fb81a66dd84b30, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:00,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/742e9da51daf4ba384fb81a66dd84b30 is 175, key is test_row_0/A:col10/1733811299838/Put/seqid=0 2024-12-10T06:15:00,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742036_1212 (size=31255) 2024-12-10T06:15:00,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811360526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811360526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811360526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811360526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811360535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T06:15:00,605 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:00,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-10T06:15:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T06:15:00,608 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:00,608 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:00,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T06:15:00,721 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=496, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/742e9da51daf4ba384fb81a66dd84b30 2024-12-10T06:15:00,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/dfe66e9223554b54b3f3be662354444b is 50, key is test_row_0/B:col10/1733811299838/Put/seqid=0 2024-12-10T06:15:00,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T06:15:00,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:00,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:15:00,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:00,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:00,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:00,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742037_1213 (size=12301) 2024-12-10T06:15:00,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/dfe66e9223554b54b3f3be662354444b 2024-12-10T06:15:00,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/da40bc60400540c79ef576288947111e is 50, key is test_row_0/C:col10/1733811299838/Put/seqid=0 2024-12-10T06:15:00,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742038_1214 (size=12301) 2024-12-10T06:15:00,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/da40bc60400540c79ef576288947111e 2024-12-10T06:15:00,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/742e9da51daf4ba384fb81a66dd84b30 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/742e9da51daf4ba384fb81a66dd84b30 2024-12-10T06:15:00,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/742e9da51daf4ba384fb81a66dd84b30, entries=150, sequenceid=496, filesize=30.5 K 2024-12-10T06:15:00,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/dfe66e9223554b54b3f3be662354444b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/dfe66e9223554b54b3f3be662354444b 2024-12-10T06:15:00,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/dfe66e9223554b54b3f3be662354444b, entries=150, sequenceid=496, filesize=12.0 K 2024-12-10T06:15:00,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/da40bc60400540c79ef576288947111e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/da40bc60400540c79ef576288947111e 2024-12-10T06:15:00,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/da40bc60400540c79ef576288947111e, entries=150, sequenceid=496, filesize=12.0 K 2024-12-10T06:15:00,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 9a8a17594d375f527e3c6e0f2fcae42e in 1007ms, sequenceid=496, compaction requested=true 2024-12-10T06:15:00,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:00,848 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:00,848 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:00,849 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:00,849 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/B is initiating minor compaction (all files) 2024-12-10T06:15:00,849 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/B in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:00,850 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2efe93e1228c485fb189c169cdaec615, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/73c001594efe412c80623588ad97b427, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/dfe66e9223554b54b3f3be662354444b] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.8 K 2024-12-10T06:15:00,851 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:00,851 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/A is initiating minor compaction (all files) 2024-12-10T06:15:00,851 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/A in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:00,851 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efe93e1228c485fb189c169cdaec615, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733811296294 2024-12-10T06:15:00,851 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/eac8d6b828b64040abec134437849d03, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/989fddddcf2d4360907f641f88127bff, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/742e9da51daf4ba384fb81a66dd84b30] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=92.3 K 2024-12-10T06:15:00,851 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:00,851 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/eac8d6b828b64040abec134437849d03, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/989fddddcf2d4360907f641f88127bff, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/742e9da51daf4ba384fb81a66dd84b30] 2024-12-10T06:15:00,852 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 73c001594efe412c80623588ad97b427, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733811296501 2024-12-10T06:15:00,852 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting eac8d6b828b64040abec134437849d03, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733811296294 2024-12-10T06:15:00,852 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe66e9223554b54b3f3be662354444b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733811297687 2024-12-10T06:15:00,852 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 989fddddcf2d4360907f641f88127bff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733811296501 2024-12-10T06:15:00,853 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 742e9da51daf4ba384fb81a66dd84b30, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733811297687 2024-12-10T06:15:00,863 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:00,864 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/794eb08e237c49f9b54307e10b3fbf2a is 50, key is test_row_0/B:col10/1733811299838/Put/seqid=0 2024-12-10T06:15:00,881 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:00,897 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121035f033a5c0f94c378425665b71ddcee5_9a8a17594d375f527e3c6e0f2fcae42e store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:00,899 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121035f033a5c0f94c378425665b71ddcee5_9a8a17594d375f527e3c6e0f2fcae42e, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:00,900 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121035f033a5c0f94c378425665b71ddcee5_9a8a17594d375f527e3c6e0f2fcae42e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:00,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T06:15:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:00,915 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T06:15:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:15:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:15:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:15:00,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T06:15:00,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742039_1215 (size=13153) 2024-12-10T06:15:00,958 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/794eb08e237c49f9b54307e10b3fbf2a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/794eb08e237c49f9b54307e10b3fbf2a 2024-12-10T06:15:00,966 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/B of 9a8a17594d375f527e3c6e0f2fcae42e into 794eb08e237c49f9b54307e10b3fbf2a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:00,966 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:00,966 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/B, priority=13, startTime=1733811300848; duration=0sec 2024-12-10T06:15:00,966 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:00,966 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:B 2024-12-10T06:15:00,966 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:00,969 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:00,969 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/C is initiating minor compaction (all files) 2024-12-10T06:15:00,969 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/C in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:00,969 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/cd8f99eed88043e5b929aaf26aa6ac27, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ec9ab93fe00a4eb9945704c953757064, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/da40bc60400540c79ef576288947111e] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.8 K 2024-12-10T06:15:00,969 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting cd8f99eed88043e5b929aaf26aa6ac27, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733811296294 2024-12-10T06:15:00,970 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ec9ab93fe00a4eb9945704c953757064, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733811296501 2024-12-10T06:15:00,971 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting da40bc60400540c79ef576288947111e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733811297687 2024-12-10T06:15:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100e47c99ea01a4d8797a0986f6a20835d_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811299884/Put/seqid=0 2024-12-10T06:15:00,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742040_1216 (size=4469) 2024-12-10T06:15:00,991 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#C#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:00,992 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/803ccb7276304bbea47b96a9b53c0b6f is 50, key is test_row_0/C:col10/1733811299838/Put/seqid=0 2024-12-10T06:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. as already flushing 2024-12-10T06:15:01,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742042_1218 (size=13153) 2024-12-10T06:15:01,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811361058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811361058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811361060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811361061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742041_1217 (size=12454) 2024-12-10T06:15:01,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811361066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:01,092 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100e47c99ea01a4d8797a0986f6a20835d_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100e47c99ea01a4d8797a0986f6a20835d_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:01,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/9990f5efd2b9492080de8e08e5ddaa34, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:01,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/9990f5efd2b9492080de8e08e5ddaa34 is 175, key is test_row_0/A:col10/1733811299884/Put/seqid=0 2024-12-10T06:15:01,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742043_1219 (size=31255) 2024-12-10T06:15:01,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811361176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811361176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811361176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811361176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811361186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T06:15:01,382 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#A#compaction#175 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:01,383 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/db0e98a5cd7c497b8b26005a866eefc6 is 175, key is test_row_0/A:col10/1733811299838/Put/seqid=0 2024-12-10T06:15:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811361385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811361386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811361386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811361386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811361406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742044_1220 (size=32107) 2024-12-10T06:15:01,429 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/db0e98a5cd7c497b8b26005a866eefc6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db0e98a5cd7c497b8b26005a866eefc6 2024-12-10T06:15:01,435 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/A of 9a8a17594d375f527e3c6e0f2fcae42e into db0e98a5cd7c497b8b26005a866eefc6(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:01,435 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:01,435 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/A, priority=13, startTime=1733811300848; duration=0sec 2024-12-10T06:15:01,435 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:01,435 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:A 2024-12-10T06:15:01,461 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/803ccb7276304bbea47b96a9b53c0b6f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/803ccb7276304bbea47b96a9b53c0b6f 2024-12-10T06:15:01,466 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/C of 9a8a17594d375f527e3c6e0f2fcae42e into 803ccb7276304bbea47b96a9b53c0b6f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:01,467 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:01,467 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/C, priority=13, startTime=1733811300848; duration=0sec 2024-12-10T06:15:01,467 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:01,467 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:C 2024-12-10T06:15:01,534 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=515, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/9990f5efd2b9492080de8e08e5ddaa34 2024-12-10T06:15:01,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/0dea3735aa5a4a4d85c67c7a76405b97 is 50, key is test_row_0/B:col10/1733811299884/Put/seqid=0 2024-12-10T06:15:01,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742045_1221 (size=12301) 2024-12-10T06:15:01,570 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/0dea3735aa5a4a4d85c67c7a76405b97 2024-12-10T06:15:01,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/6d04a9eb9f3e4826a2dbf5ebbf21dd53 is 50, key is test_row_0/C:col10/1733811299884/Put/seqid=0 2024-12-10T06:15:01,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742046_1222 (size=12301) 2024-12-10T06:15:01,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811361696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811361696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811361696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811361696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811361715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T06:15:01,999 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/6d04a9eb9f3e4826a2dbf5ebbf21dd53 2024-12-10T06:15:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/9990f5efd2b9492080de8e08e5ddaa34 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9990f5efd2b9492080de8e08e5ddaa34 2024-12-10T06:15:02,009 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9990f5efd2b9492080de8e08e5ddaa34, entries=150, sequenceid=515, filesize=30.5 K 2024-12-10T06:15:02,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/0dea3735aa5a4a4d85c67c7a76405b97 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0dea3735aa5a4a4d85c67c7a76405b97 2024-12-10T06:15:02,015 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0dea3735aa5a4a4d85c67c7a76405b97, entries=150, sequenceid=515, filesize=12.0 K 2024-12-10T06:15:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/6d04a9eb9f3e4826a2dbf5ebbf21dd53 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/6d04a9eb9f3e4826a2dbf5ebbf21dd53 2024-12-10T06:15:02,022 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/6d04a9eb9f3e4826a2dbf5ebbf21dd53, entries=150, sequenceid=515, filesize=12.0 K 2024-12-10T06:15:02,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9a8a17594d375f527e3c6e0f2fcae42e in 1108ms, sequenceid=515, compaction requested=false 2024-12-10T06:15:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-10T06:15:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-10T06:15:02,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-10T06:15:02,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-12-10T06:15:02,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4210 sec 2024-12-10T06:15:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:02,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T06:15:02,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:15:02,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:15:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:15:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:02,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109c19614a0245472e8616b9df7a307c1b_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811301056/Put/seqid=0 2024-12-10T06:15:02,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742047_1223 (size=17534) 2024-12-10T06:15:02,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811362233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811362235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811362236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811362238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811362239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811362346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811362346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811362355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811362355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811362355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1733811362556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733811362556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43848 deadline: 1733811362565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43908 deadline: 1733811362565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43884 deadline: 1733811362565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:02,636 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:02,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109c19614a0245472e8616b9df7a307c1b_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109c19614a0245472e8616b9df7a307c1b_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:02,653 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/560225e3c0d0436a9bbab7c57b478013, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:02,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/560225e3c0d0436a9bbab7c57b478013 is 175, key is test_row_0/A:col10/1733811301056/Put/seqid=0 2024-12-10T06:15:02,679 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-12-10T06:15:02,679 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:02,679 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,679 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,688 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-12-10T06:15:02,688 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:02,688 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,688 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742048_1224 (size=48639) 2024-12-10T06:15:02,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=535, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/560225e3c0d0436a9bbab7c57b478013 2024-12-10T06:15:02,694 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-12-10T06:15:02,694 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:02,694 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,695 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,701 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-12-10T06:15:02,701 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:02,701 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,701 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/8ecc037e787e4a77a9b2f695d080cc95 is 50, key is test_row_0/B:col10/1733811301056/Put/seqid=0 2024-12-10T06:15:02,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742049_1225 (size=12301) 2024-12-10T06:15:02,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/8ecc037e787e4a77a9b2f695d080cc95 2024-12-10T06:15:02,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5774f0a82b9a434e877ba9eb07439b85 is 50, key is test_row_0/C:col10/1733811301056/Put/seqid=0 2024-12-10T06:15:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T06:15:02,756 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:02,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742050_1226 (size=12301) 2024-12-10T06:15:02,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5774f0a82b9a434e877ba9eb07439b85 2024-12-10T06:15:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/560225e3c0d0436a9bbab7c57b478013 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/560225e3c0d0436a9bbab7c57b478013 2024-12-10T06:15:02,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/560225e3c0d0436a9bbab7c57b478013, entries=250, sequenceid=535, filesize=47.5 K 2024-12-10T06:15:02,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/8ecc037e787e4a77a9b2f695d080cc95 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8ecc037e787e4a77a9b2f695d080cc95 2024-12-10T06:15:02,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8ecc037e787e4a77a9b2f695d080cc95, entries=150, sequenceid=535, filesize=12.0 K 2024-12-10T06:15:02,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/5774f0a82b9a434e877ba9eb07439b85 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5774f0a82b9a434e877ba9eb07439b85 2024-12-10T06:15:02,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5774f0a82b9a434e877ba9eb07439b85, entries=150, sequenceid=535, filesize=12.0 K 2024-12-10T06:15:02,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9a8a17594d375f527e3c6e0f2fcae42e in 595ms, sequenceid=535, compaction requested=true 2024-12-10T06:15:02,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:02,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:02,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:02,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:02,804 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:02,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:02,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a8a17594d375f527e3c6e0f2fcae42e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:02,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T06:15:02,804 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:02,806 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:02,806 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/A is initiating minor compaction (all files) 2024-12-10T06:15:02,806 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/A in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:02,806 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db0e98a5cd7c497b8b26005a866eefc6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9990f5efd2b9492080de8e08e5ddaa34, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/560225e3c0d0436a9bbab7c57b478013] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=109.4 K 2024-12-10T06:15:02,806 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:02,806 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db0e98a5cd7c497b8b26005a866eefc6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9990f5efd2b9492080de8e08e5ddaa34, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/560225e3c0d0436a9bbab7c57b478013] 2024-12-10T06:15:02,807 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:02,807 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/B is initiating minor compaction (all files) 2024-12-10T06:15:02,807 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/B in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:02,807 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/794eb08e237c49f9b54307e10b3fbf2a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0dea3735aa5a4a4d85c67c7a76405b97, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8ecc037e787e4a77a9b2f695d080cc95] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.9 K 2024-12-10T06:15:02,808 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 794eb08e237c49f9b54307e10b3fbf2a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733811297687 2024-12-10T06:15:02,808 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting db0e98a5cd7c497b8b26005a866eefc6, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733811297687 2024-12-10T06:15:02,809 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9990f5efd2b9492080de8e08e5ddaa34, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1733811299884 2024-12-10T06:15:02,809 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 560225e3c0d0436a9bbab7c57b478013, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733811301056 2024-12-10T06:15:02,809 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dea3735aa5a4a4d85c67c7a76405b97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1733811299884 2024-12-10T06:15:02,810 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ecc037e787e4a77a9b2f695d080cc95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733811301056 2024-12-10T06:15:02,833 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#B#compaction#183 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:02,834 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/a9192bd362b44a2cb30a649c1be2b18f is 50, key is test_row_0/B:col10/1733811301056/Put/seqid=0 2024-12-10T06:15:02,845 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:02,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742051_1227 (size=13255) 2024-12-10T06:15:02,864 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210a1d099457a6043d49bc022e20832f9ec_9a8a17594d375f527e3c6e0f2fcae42e store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:02,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:15:02,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:15:02,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:02,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:15:02,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:02,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:15:02,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:02,872 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-12-10T06:15:02,872 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:02,872 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,872 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-12-10T06:15:02,872 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:02,872 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,872 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,872 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,879 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-12-10T06:15:02,880 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:02,880 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,880 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,880 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-12-10T06:15:02,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,880 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:02,880 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,880 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121069aa1382dbe346c7abd4a947f75fdb62_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811302868/Put/seqid=0 2024-12-10T06:15:02,892 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-12-10T06:15:02,893 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:02,893 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,893 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2223 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2245 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 993 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2977 rows 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 993 2024-12-10T06:15:02,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2978 rows 2024-12-10T06:15:02,893 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:15:02,894 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:15:02,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:02,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:02,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T06:15:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T06:15:02,901 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811302900"}]},"ts":"1733811302900"} 2024-12-10T06:15:02,903 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T06:15:02,903 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T06:15:02,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T06:15:02,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, UNASSIGN}] 2024-12-10T06:15:02,907 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, UNASSIGN 2024-12-10T06:15:02,908 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=9a8a17594d375f527e3c6e0f2fcae42e, regionState=CLOSING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:02,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, UNASSIGN because future has completed 2024-12-10T06:15:02,911 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:15:02,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:15:02,921 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210a1d099457a6043d49bc022e20832f9ec_9a8a17594d375f527e3c6e0f2fcae42e, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:02,921 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a1d099457a6043d49bc022e20832f9ec_9a8a17594d375f527e3c6e0f2fcae42e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:02,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742052_1228 (size=12454) 2024-12-10T06:15:02,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742053_1229 (size=4469) 2024-12-10T06:15:02,996 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#A#compaction#184 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:02,998 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/15ba38b10e554d8bbb276b8630dfa676 is 175, key is test_row_0/A:col10/1733811301056/Put/seqid=0 2024-12-10T06:15:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T06:15:03,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742054_1230 (size=32209) 2024-12-10T06:15:03,034 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/15ba38b10e554d8bbb276b8630dfa676 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/15ba38b10e554d8bbb276b8630dfa676 2024-12-10T06:15:03,042 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/A of 9a8a17594d375f527e3c6e0f2fcae42e into 15ba38b10e554d8bbb276b8630dfa676(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:03,042 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:03,042 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/A, priority=13, startTime=1733811302804; duration=0sec 2024-12-10T06:15:03,042 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:03,042 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:A 2024-12-10T06:15:03,042 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:03,044 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:03,044 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 9a8a17594d375f527e3c6e0f2fcae42e/C is initiating minor compaction (all files) 2024-12-10T06:15:03,044 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a8a17594d375f527e3c6e0f2fcae42e/C in TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:03,044 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/803ccb7276304bbea47b96a9b53c0b6f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/6d04a9eb9f3e4826a2dbf5ebbf21dd53, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5774f0a82b9a434e877ba9eb07439b85] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp, totalSize=36.9 K 2024-12-10T06:15:03,045 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 803ccb7276304bbea47b96a9b53c0b6f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733811297687 2024-12-10T06:15:03,046 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d04a9eb9f3e4826a2dbf5ebbf21dd53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1733811299884 2024-12-10T06:15:03,047 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 5774f0a82b9a434e877ba9eb07439b85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733811301056 2024-12-10T06:15:03,060 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a8a17594d375f527e3c6e0f2fcae42e#C#compaction#186 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:03,061 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/fd36a92445db42bd89b84ab6112a9c20 is 50, key is test_row_0/C:col10/1733811301056/Put/seqid=0 2024-12-10T06:15:03,065 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:03,065 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:15:03,065 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 9a8a17594d375f527e3c6e0f2fcae42e, disabling compactions & flushes 2024-12-10T06:15:03,065 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1993): waiting for 2 compactions & cache flush to complete for region TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:03,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742055_1231 (size=13255) 2024-12-10T06:15:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T06:15:03,269 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/a9192bd362b44a2cb30a649c1be2b18f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/a9192bd362b44a2cb30a649c1be2b18f 2024-12-10T06:15:03,275 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/B of 9a8a17594d375f527e3c6e0f2fcae42e into a9192bd362b44a2cb30a649c1be2b18f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:03,275 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:03,275 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/B, priority=13, startTime=1733811302804; duration=0sec 2024-12-10T06:15:03,275 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:03,275 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:B 2024-12-10T06:15:03,352 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:03,355 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121069aa1382dbe346c7abd4a947f75fdb62_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121069aa1382dbe346c7abd4a947f75fdb62_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:03,356 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/31017bd0fb034e1f92433f7aeab989f7, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:03,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/31017bd0fb034e1f92433f7aeab989f7 is 175, key is test_row_0/A:col10/1733811302868/Put/seqid=0 2024-12-10T06:15:03,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742056_1232 (size=31255) 2024-12-10T06:15:03,508 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/fd36a92445db42bd89b84ab6112a9c20 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/fd36a92445db42bd89b84ab6112a9c20 2024-12-10T06:15:03,512 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a8a17594d375f527e3c6e0f2fcae42e/C of 9a8a17594d375f527e3c6e0f2fcae42e into fd36a92445db42bd89b84ab6112a9c20(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:03,512 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:03,512 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:03,512 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., storeName=9a8a17594d375f527e3c6e0f2fcae42e/C, priority=13, startTime=1733811302804; duration=0sec 2024-12-10T06:15:03,512 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:03,512 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a8a17594d375f527e3c6e0f2fcae42e:C 2024-12-10T06:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T06:15:03,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=552, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/31017bd0fb034e1f92433f7aeab989f7 2024-12-10T06:15:03,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/c59f4bdb29c24e408fa4bd8c031bdd7f is 50, key is test_row_0/B:col10/1733811302868/Put/seqid=0 2024-12-10T06:15:03,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742057_1233 (size=12301) 2024-12-10T06:15:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T06:15:04,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/c59f4bdb29c24e408fa4bd8c031bdd7f 2024-12-10T06:15:04,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/9d5154847cdc4a8e9ae82c2414ec0366 is 50, key is test_row_0/C:col10/1733811302868/Put/seqid=0 2024-12-10T06:15:04,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742058_1234 (size=12301) 2024-12-10T06:15:04,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/9d5154847cdc4a8e9ae82c2414ec0366 2024-12-10T06:15:04,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/31017bd0fb034e1f92433f7aeab989f7 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/31017bd0fb034e1f92433f7aeab989f7 2024-12-10T06:15:04,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/31017bd0fb034e1f92433f7aeab989f7, entries=150, sequenceid=552, filesize=30.5 K 2024-12-10T06:15:04,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/c59f4bdb29c24e408fa4bd8c031bdd7f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/c59f4bdb29c24e408fa4bd8c031bdd7f 2024-12-10T06:15:04,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/c59f4bdb29c24e408fa4bd8c031bdd7f, entries=150, sequenceid=552, filesize=12.0 K 2024-12-10T06:15:04,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/9d5154847cdc4a8e9ae82c2414ec0366 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/9d5154847cdc4a8e9ae82c2414ec0366 2024-12-10T06:15:04,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/9d5154847cdc4a8e9ae82c2414ec0366, entries=150, sequenceid=552, filesize=12.0 K 2024-12-10T06:15:04,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 9a8a17594d375f527e3c6e0f2fcae42e in 1735ms, sequenceid=552, compaction requested=false 2024-12-10T06:15:04,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a8a17594d375f527e3c6e0f2fcae42e: 2024-12-10T06:15:04,605 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:04,605 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:04,605 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. after waiting 0 ms 2024-12-10T06:15:04,605 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:04,605 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 9a8a17594d375f527e3c6e0f2fcae42e 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-10T06:15:04,605 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=A 2024-12-10T06:15:04,605 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:04,605 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=B 2024-12-10T06:15:04,605 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:04,605 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a8a17594d375f527e3c6e0f2fcae42e, store=C 2024-12-10T06:15:04,605 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:04,610 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121042ff18ae2e1143f19364598cfd9702fc_9a8a17594d375f527e3c6e0f2fcae42e is 50, key is test_row_0/A:col10/1733811302886/Put/seqid=0 2024-12-10T06:15:04,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742059_1235 (size=9914) 2024-12-10T06:15:05,016 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:05,020 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121042ff18ae2e1143f19364598cfd9702fc_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121042ff18ae2e1143f19364598cfd9702fc_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:05,020 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/89c8203ac6ce4d23b9c51d07d11659ff, store: [table=TestAcidGuarantees family=A region=9a8a17594d375f527e3c6e0f2fcae42e] 2024-12-10T06:15:05,021 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/89c8203ac6ce4d23b9c51d07d11659ff is 175, key is test_row_0/A:col10/1733811302886/Put/seqid=0 2024-12-10T06:15:05,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742060_1236 (size=22561) 2024-12-10T06:15:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T06:15:05,425 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=561, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/89c8203ac6ce4d23b9c51d07d11659ff 2024-12-10T06:15:05,435 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/a8a0971e607d43988abf119e76489ed6 is 50, key is test_row_0/B:col10/1733811302886/Put/seqid=0 2024-12-10T06:15:05,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742061_1237 (size=9857) 2024-12-10T06:15:05,450 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/a8a0971e607d43988abf119e76489ed6 2024-12-10T06:15:05,461 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/07003007304e49cda3c4a70cce83f1aa is 50, key is test_row_0/C:col10/1733811302886/Put/seqid=0 2024-12-10T06:15:05,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742062_1238 (size=9857) 2024-12-10T06:15:05,472 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/07003007304e49cda3c4a70cce83f1aa 2024-12-10T06:15:05,479 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/A/89c8203ac6ce4d23b9c51d07d11659ff as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/89c8203ac6ce4d23b9c51d07d11659ff 2024-12-10T06:15:05,484 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/89c8203ac6ce4d23b9c51d07d11659ff, entries=100, sequenceid=561, filesize=22.0 K 2024-12-10T06:15:05,486 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/B/a8a0971e607d43988abf119e76489ed6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/a8a0971e607d43988abf119e76489ed6 2024-12-10T06:15:05,491 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/a8a0971e607d43988abf119e76489ed6, entries=100, sequenceid=561, filesize=9.6 K 2024-12-10T06:15:05,492 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/.tmp/C/07003007304e49cda3c4a70cce83f1aa as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/07003007304e49cda3c4a70cce83f1aa 2024-12-10T06:15:05,496 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/07003007304e49cda3c4a70cce83f1aa, entries=100, sequenceid=561, filesize=9.6 K 2024-12-10T06:15:05,497 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9a8a17594d375f527e3c6e0f2fcae42e in 892ms, sequenceid=561, compaction requested=true 2024-12-10T06:15:05,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9cbbf7bc60464ebe93647de55500b860, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b917f9a138824e1cbd0cd30a5c73d1ce, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/f632278a6b0e4a93b5f9879918e0fd81, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/3e51f3cf46a543bd96676d5603967063, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ff35cfe62eff4aaf9fd76f80b34c23e3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/7747553791bd4adfa2c84e7eb6e43292, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/8c89af72308a43b4b43f41379d0776f5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5fa89e3300774625b46ef86fa97d3887, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/88d6bb788555412da75207c8b83b6ca3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ebd4097964f64eb9b56f7f7274d17125, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/6deb6e78facf49ceb38d81b2986f5891, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b76fa1283c9e4d2c9dcb363d86826c16, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/567f51bad1464eb5ab2937c801b248e2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/c4eceebbc2c94cee8b2b2540730a2856, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5ef9b85346454857a1b20faa4aa5aa15, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0bf20b972fd04784a755520c71eafa57, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ad9e560b316e4e4dbe3d33fbdefeffde, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0e9b9afc83b444959fd97bf2f123ba13, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/10e89bb23ff74c5492d5cc99be44ec24, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/05a7b01ed8f74952a7d37d7e6b177799, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db144b06a4bc44c39a3dbdce2003d5ed, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/032b57fdc7384cefa3a68e6016d7c0e6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/eac8d6b828b64040abec134437849d03, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/989fddddcf2d4360907f641f88127bff, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db0e98a5cd7c497b8b26005a866eefc6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/742e9da51daf4ba384fb81a66dd84b30, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9990f5efd2b9492080de8e08e5ddaa34, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/560225e3c0d0436a9bbab7c57b478013] to archive 2024-12-10T06:15:05,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:15:05,508 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9cbbf7bc60464ebe93647de55500b860 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9cbbf7bc60464ebe93647de55500b860 2024-12-10T06:15:05,511 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ff35cfe62eff4aaf9fd76f80b34c23e3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ff35cfe62eff4aaf9fd76f80b34c23e3 2024-12-10T06:15:05,511 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5fa89e3300774625b46ef86fa97d3887 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5fa89e3300774625b46ef86fa97d3887 2024-12-10T06:15:05,511 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/8c89af72308a43b4b43f41379d0776f5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/8c89af72308a43b4b43f41379d0776f5 2024-12-10T06:15:05,512 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/3e51f3cf46a543bd96676d5603967063 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/3e51f3cf46a543bd96676d5603967063 2024-12-10T06:15:05,512 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/f632278a6b0e4a93b5f9879918e0fd81 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/f632278a6b0e4a93b5f9879918e0fd81 2024-12-10T06:15:05,512 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/7747553791bd4adfa2c84e7eb6e43292 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/7747553791bd4adfa2c84e7eb6e43292 2024-12-10T06:15:05,512 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b917f9a138824e1cbd0cd30a5c73d1ce to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b917f9a138824e1cbd0cd30a5c73d1ce 2024-12-10T06:15:05,515 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/88d6bb788555412da75207c8b83b6ca3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/88d6bb788555412da75207c8b83b6ca3 2024-12-10T06:15:05,516 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/6deb6e78facf49ceb38d81b2986f5891 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/6deb6e78facf49ceb38d81b2986f5891 2024-12-10T06:15:05,517 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/567f51bad1464eb5ab2937c801b248e2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/567f51bad1464eb5ab2937c801b248e2 2024-12-10T06:15:05,517 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b76fa1283c9e4d2c9dcb363d86826c16 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/b76fa1283c9e4d2c9dcb363d86826c16 2024-12-10T06:15:05,517 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/c4eceebbc2c94cee8b2b2540730a2856 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/c4eceebbc2c94cee8b2b2540730a2856 2024-12-10T06:15:05,518 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ebd4097964f64eb9b56f7f7274d17125 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ebd4097964f64eb9b56f7f7274d17125 2024-12-10T06:15:05,518 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5ef9b85346454857a1b20faa4aa5aa15 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/5ef9b85346454857a1b20faa4aa5aa15 2024-12-10T06:15:05,519 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0bf20b972fd04784a755520c71eafa57 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0bf20b972fd04784a755520c71eafa57 2024-12-10T06:15:05,521 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ad9e560b316e4e4dbe3d33fbdefeffde to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/ad9e560b316e4e4dbe3d33fbdefeffde 2024-12-10T06:15:05,521 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0e9b9afc83b444959fd97bf2f123ba13 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/0e9b9afc83b444959fd97bf2f123ba13 2024-12-10T06:15:05,522 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/10e89bb23ff74c5492d5cc99be44ec24 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/10e89bb23ff74c5492d5cc99be44ec24 2024-12-10T06:15:05,523 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/032b57fdc7384cefa3a68e6016d7c0e6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/032b57fdc7384cefa3a68e6016d7c0e6 2024-12-10T06:15:05,523 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db144b06a4bc44c39a3dbdce2003d5ed to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db144b06a4bc44c39a3dbdce2003d5ed 2024-12-10T06:15:05,523 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/eac8d6b828b64040abec134437849d03 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/eac8d6b828b64040abec134437849d03 2024-12-10T06:15:05,523 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/989fddddcf2d4360907f641f88127bff to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/989fddddcf2d4360907f641f88127bff 2024-12-10T06:15:05,523 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/05a7b01ed8f74952a7d37d7e6b177799 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/05a7b01ed8f74952a7d37d7e6b177799 2024-12-10T06:15:05,525 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db0e98a5cd7c497b8b26005a866eefc6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/db0e98a5cd7c497b8b26005a866eefc6 2024-12-10T06:15:05,525 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9990f5efd2b9492080de8e08e5ddaa34 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/9990f5efd2b9492080de8e08e5ddaa34 2024-12-10T06:15:05,525 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/742e9da51daf4ba384fb81a66dd84b30 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/742e9da51daf4ba384fb81a66dd84b30 2024-12-10T06:15:05,526 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/560225e3c0d0436a9bbab7c57b478013 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/560225e3c0d0436a9bbab7c57b478013 2024-12-10T06:15:05,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/9be4bd475fe4452080df1446b33c7059, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/5a65a4dcd6564952abfe365b8f19921e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e210880f98cd4f5b906b9cae1d521ad6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/17e63a699a0b4e83ade74042127a4ce9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3578e82e9c58407db7ec6d5407425dd2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/cec5a6de9c9a43ddb77ed596bf98db42, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/4ac51d6c716545dfafc15aedb582bbb0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/82fa0ad53f134891be98a7c77e31d0e9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/aaf15bbc329a40be9740c49ba55c52de, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3faf0c21622940b9830d567e3370239e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0808f1fa132d4f6b8b48718aa294b62d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ce84652a772748bda6b6643f215f7b79, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/09fffc6cc61d4483bd32911a9884767d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8064da6de08e4d07b32d91fb81337fdc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/da8ea34f71a2441fbabe68a0260929d9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/7ec47e4f2c074feb93e9e277e68b2e45, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/740f5f0e4e6d4d0a81149f07cf698005, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e2f5248b9b3148aabcbc5be6695e2a7d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2e3e6d2e4ff74dddb347d9c8d1d6f7b4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/f1fb5a85988c4c2faeb7b010ff7a339c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ac95dffd03b949479b8c1f580724f8ec, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2efe93e1228c485fb189c169cdaec615, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/b66dcabee71140dba471e720a6c6638a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/73c001594efe412c80623588ad97b427, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/794eb08e237c49f9b54307e10b3fbf2a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/dfe66e9223554b54b3f3be662354444b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0dea3735aa5a4a4d85c67c7a76405b97, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8ecc037e787e4a77a9b2f695d080cc95] to archive 2024-12-10T06:15:05,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:15:05,538 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/82fa0ad53f134891be98a7c77e31d0e9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/82fa0ad53f134891be98a7c77e31d0e9 2024-12-10T06:15:05,538 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/5a65a4dcd6564952abfe365b8f19921e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/5a65a4dcd6564952abfe365b8f19921e 2024-12-10T06:15:05,538 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/cec5a6de9c9a43ddb77ed596bf98db42 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/cec5a6de9c9a43ddb77ed596bf98db42 2024-12-10T06:15:05,539 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/4ac51d6c716545dfafc15aedb582bbb0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/4ac51d6c716545dfafc15aedb582bbb0 2024-12-10T06:15:05,539 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e210880f98cd4f5b906b9cae1d521ad6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e210880f98cd4f5b906b9cae1d521ad6 2024-12-10T06:15:05,539 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/9be4bd475fe4452080df1446b33c7059 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/9be4bd475fe4452080df1446b33c7059 2024-12-10T06:15:05,539 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3578e82e9c58407db7ec6d5407425dd2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3578e82e9c58407db7ec6d5407425dd2 2024-12-10T06:15:05,539 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/17e63a699a0b4e83ade74042127a4ce9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/17e63a699a0b4e83ade74042127a4ce9 2024-12-10T06:15:05,545 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/da8ea34f71a2441fbabe68a0260929d9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/da8ea34f71a2441fbabe68a0260929d9 2024-12-10T06:15:05,545 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0808f1fa132d4f6b8b48718aa294b62d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0808f1fa132d4f6b8b48718aa294b62d 2024-12-10T06:15:05,545 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3faf0c21622940b9830d567e3370239e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/3faf0c21622940b9830d567e3370239e 2024-12-10T06:15:05,545 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/09fffc6cc61d4483bd32911a9884767d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/09fffc6cc61d4483bd32911a9884767d 2024-12-10T06:15:05,545 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/aaf15bbc329a40be9740c49ba55c52de to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/aaf15bbc329a40be9740c49ba55c52de 2024-12-10T06:15:05,545 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ce84652a772748bda6b6643f215f7b79 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ce84652a772748bda6b6643f215f7b79 2024-12-10T06:15:05,545 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/7ec47e4f2c074feb93e9e277e68b2e45 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/7ec47e4f2c074feb93e9e277e68b2e45 2024-12-10T06:15:05,545 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8064da6de08e4d07b32d91fb81337fdc to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8064da6de08e4d07b32d91fb81337fdc 2024-12-10T06:15:05,547 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/740f5f0e4e6d4d0a81149f07cf698005 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/740f5f0e4e6d4d0a81149f07cf698005 2024-12-10T06:15:05,548 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e2f5248b9b3148aabcbc5be6695e2a7d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/e2f5248b9b3148aabcbc5be6695e2a7d 2024-12-10T06:15:05,548 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/f1fb5a85988c4c2faeb7b010ff7a339c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/f1fb5a85988c4c2faeb7b010ff7a339c 2024-12-10T06:15:05,548 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2efe93e1228c485fb189c169cdaec615 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2efe93e1228c485fb189c169cdaec615 2024-12-10T06:15:05,548 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2e3e6d2e4ff74dddb347d9c8d1d6f7b4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/2e3e6d2e4ff74dddb347d9c8d1d6f7b4 2024-12-10T06:15:05,548 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ac95dffd03b949479b8c1f580724f8ec to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/ac95dffd03b949479b8c1f580724f8ec 2024-12-10T06:15:05,548 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/b66dcabee71140dba471e720a6c6638a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/b66dcabee71140dba471e720a6c6638a 2024-12-10T06:15:05,549 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/73c001594efe412c80623588ad97b427 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/73c001594efe412c80623588ad97b427 2024-12-10T06:15:05,551 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/794eb08e237c49f9b54307e10b3fbf2a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/794eb08e237c49f9b54307e10b3fbf2a 2024-12-10T06:15:05,552 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/dfe66e9223554b54b3f3be662354444b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/dfe66e9223554b54b3f3be662354444b 2024-12-10T06:15:05,552 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0dea3735aa5a4a4d85c67c7a76405b97 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/0dea3735aa5a4a4d85c67c7a76405b97 2024-12-10T06:15:05,552 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8ecc037e787e4a77a9b2f695d080cc95 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/8ecc037e787e4a77a9b2f695d080cc95 2024-12-10T06:15:05,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f660f545d09c47df9b9a126b9de38bd0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8ff9781adb5347e0a39438900d99d876, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/3a7a98c65519457abef9e5a62d55b8e2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/45fe9d855f834ada819708b826a86c53, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8328d8038b244b29af3431654fe8733b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/72d00a783e7646169bf4a8b64f6a5d48, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ad76fda382e34fe887834bf00a0e1b03, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/565b19fe99b34d3b9ffafe9cf04a24e5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b1fdefd2392b4b58ac5ecce004757a99, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/01246ab14f3b4349ad36c53e0634824e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/d1e76290deed4e8f952c94faf53ad2e4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/32806055834045878b86d0c6d38387c1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/0d60af3961ab45df8d9660397da0def6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b749167b0b6e4592a270ef42c287954f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5874e1f75be0401cb9b35437edbcc014, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7d5e5959a49d45fa9551412fc554c78a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f2f21c644e2146f68befc2896ec77631, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f24a35a831824373ba4a1cf900ca533b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5a9aee124e7f4000abf90f849c8712d5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5deee539a9cb42a1ab88312c129c665a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7489faf972a74d27bb1755c85ce08b96, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/cd8f99eed88043e5b929aaf26aa6ac27, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/18dcbb92e9684007ab4e3441c4f84b67, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ec9ab93fe00a4eb9945704c953757064, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/803ccb7276304bbea47b96a9b53c0b6f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/da40bc60400540c79ef576288947111e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/6d04a9eb9f3e4826a2dbf5ebbf21dd53, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5774f0a82b9a434e877ba9eb07439b85] to archive 2024-12-10T06:15:05,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:15:05,565 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/3a7a98c65519457abef9e5a62d55b8e2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/3a7a98c65519457abef9e5a62d55b8e2 2024-12-10T06:15:05,565 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/45fe9d855f834ada819708b826a86c53 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/45fe9d855f834ada819708b826a86c53 2024-12-10T06:15:05,565 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8ff9781adb5347e0a39438900d99d876 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8ff9781adb5347e0a39438900d99d876 2024-12-10T06:15:05,565 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/565b19fe99b34d3b9ffafe9cf04a24e5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/565b19fe99b34d3b9ffafe9cf04a24e5 2024-12-10T06:15:05,565 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8328d8038b244b29af3431654fe8733b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/8328d8038b244b29af3431654fe8733b 2024-12-10T06:15:05,566 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ad76fda382e34fe887834bf00a0e1b03 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ad76fda382e34fe887834bf00a0e1b03 2024-12-10T06:15:05,566 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/72d00a783e7646169bf4a8b64f6a5d48 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/72d00a783e7646169bf4a8b64f6a5d48 2024-12-10T06:15:05,566 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f660f545d09c47df9b9a126b9de38bd0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f660f545d09c47df9b9a126b9de38bd0 2024-12-10T06:15:05,567 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b1fdefd2392b4b58ac5ecce004757a99 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b1fdefd2392b4b58ac5ecce004757a99 2024-12-10T06:15:05,568 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/d1e76290deed4e8f952c94faf53ad2e4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/d1e76290deed4e8f952c94faf53ad2e4 2024-12-10T06:15:05,568 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5874e1f75be0401cb9b35437edbcc014 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5874e1f75be0401cb9b35437edbcc014 2024-12-10T06:15:05,568 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/01246ab14f3b4349ad36c53e0634824e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/01246ab14f3b4349ad36c53e0634824e 2024-12-10T06:15:05,568 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/0d60af3961ab45df8d9660397da0def6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/0d60af3961ab45df8d9660397da0def6 2024-12-10T06:15:05,568 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/32806055834045878b86d0c6d38387c1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/32806055834045878b86d0c6d38387c1 2024-12-10T06:15:05,569 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b749167b0b6e4592a270ef42c287954f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/b749167b0b6e4592a270ef42c287954f 2024-12-10T06:15:05,569 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7d5e5959a49d45fa9551412fc554c78a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7d5e5959a49d45fa9551412fc554c78a 2024-12-10T06:15:05,573 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5deee539a9cb42a1ab88312c129c665a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5deee539a9cb42a1ab88312c129c665a 2024-12-10T06:15:05,573 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f2f21c644e2146f68befc2896ec77631 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f2f21c644e2146f68befc2896ec77631 2024-12-10T06:15:05,573 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/18dcbb92e9684007ab4e3441c4f84b67 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/18dcbb92e9684007ab4e3441c4f84b67 2024-12-10T06:15:05,573 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5a9aee124e7f4000abf90f849c8712d5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5a9aee124e7f4000abf90f849c8712d5 2024-12-10T06:15:05,574 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7489faf972a74d27bb1755c85ce08b96 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/7489faf972a74d27bb1755c85ce08b96 2024-12-10T06:15:05,574 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/cd8f99eed88043e5b929aaf26aa6ac27 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/cd8f99eed88043e5b929aaf26aa6ac27 2024-12-10T06:15:05,574 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f24a35a831824373ba4a1cf900ca533b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/f24a35a831824373ba4a1cf900ca533b 2024-12-10T06:15:05,575 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5774f0a82b9a434e877ba9eb07439b85 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/5774f0a82b9a434e877ba9eb07439b85 2024-12-10T06:15:05,576 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/6d04a9eb9f3e4826a2dbf5ebbf21dd53 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/6d04a9eb9f3e4826a2dbf5ebbf21dd53 2024-12-10T06:15:05,576 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/803ccb7276304bbea47b96a9b53c0b6f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/803ccb7276304bbea47b96a9b53c0b6f 2024-12-10T06:15:05,576 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/da40bc60400540c79ef576288947111e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/da40bc60400540c79ef576288947111e 2024-12-10T06:15:05,577 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ec9ab93fe00a4eb9945704c953757064 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/ec9ab93fe00a4eb9945704c953757064 2024-12-10T06:15:05,585 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/recovered.edits/564.seqid, newMaxSeqId=564, maxSeqId=4 2024-12-10T06:15:05,586 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. 2024-12-10T06:15:05,587 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 9a8a17594d375f527e3c6e0f2fcae42e: Waiting for close lock at 1733811303065Running coprocessor pre-close hooks at 1733811303065Disabling compacts and flushes for region at 1733811303065Disabling writes for close at 1733811304605 (+1540 ms)Obtaining lock to block concurrent updates at 1733811304605Preparing flush snapshotting stores in 9a8a17594d375f527e3c6e0f2fcae42e at 1733811304605Finished memstore snapshotting TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733811304605Flushing stores of TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e. at 1733811304606 (+1 ms)Flushing 9a8a17594d375f527e3c6e0f2fcae42e/A: creating writer at 1733811304606Flushing mob file 9a8a17594d375f527e3c6e0f2fcae42e/A: appending metadata at 1733811304610 (+4 ms)Flushing mob file 9a8a17594d375f527e3c6e0f2fcae42e/A: closing flushed file at 1733811304610Flushing 9a8a17594d375f527e3c6e0f2fcae42e/A: appending metadata at 1733811305020 (+410 ms)Flushing 9a8a17594d375f527e3c6e0f2fcae42e/A: closing flushed file at 1733811305020Flushing 9a8a17594d375f527e3c6e0f2fcae42e/B: creating writer at 1733811305431 (+411 ms)Flushing 9a8a17594d375f527e3c6e0f2fcae42e/B: appending metadata at 1733811305435 (+4 ms)Flushing 9a8a17594d375f527e3c6e0f2fcae42e/B: closing flushed file at 1733811305435Flushing 9a8a17594d375f527e3c6e0f2fcae42e/C: creating writer at 1733811305457 (+22 ms)Flushing 9a8a17594d375f527e3c6e0f2fcae42e/C: appending metadata at 1733811305461 (+4 ms)Flushing 9a8a17594d375f527e3c6e0f2fcae42e/C: closing flushed file at 1733811305461Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a457cc6: reopening flushed file at 1733811305478 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c1c264b: reopening flushed file at 1733811305484 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c99a176: reopening flushed file at 1733811305491 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9a8a17594d375f527e3c6e0f2fcae42e in 892ms, sequenceid=561, compaction requested=true at 1733811305497 (+6 ms)Writing region close event to WAL at 1733811305579 (+82 ms)Running coprocessor post-close hooks at 1733811305586 (+7 ms)Closed at 1733811305586 2024-12-10T06:15:05,589 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:05,589 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=9a8a17594d375f527e3c6e0f2fcae42e, regionState=CLOSED 2024-12-10T06:15:05,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:15:05,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-10T06:15:05,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 9a8a17594d375f527e3c6e0f2fcae42e, server=c42c682b34f7,43983,1733811247584 in 2.6800 sec 2024-12-10T06:15:05,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-12-10T06:15:05,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a8a17594d375f527e3c6e0f2fcae42e, UNASSIGN in 2.6880 sec 2024-12-10T06:15:05,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-10T06:15:05,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6930 sec 2024-12-10T06:15:05,601 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811305600"}]},"ts":"1733811305600"} 2024-12-10T06:15:05,602 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T06:15:05,602 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T06:15:05,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7070 sec 2024-12-10T06:15:05,915 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T06:15:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T06:15:07,066 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:07,067 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T06:15:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:07,069 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:07,069 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T06:15:07,072 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,075 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/recovered.edits] 2024-12-10T06:15:07,079 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/15ba38b10e554d8bbb276b8630dfa676 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/15ba38b10e554d8bbb276b8630dfa676 2024-12-10T06:15:07,079 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/89c8203ac6ce4d23b9c51d07d11659ff to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/89c8203ac6ce4d23b9c51d07d11659ff 2024-12-10T06:15:07,079 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/31017bd0fb034e1f92433f7aeab989f7 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/A/31017bd0fb034e1f92433f7aeab989f7 2024-12-10T06:15:07,083 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/a8a0971e607d43988abf119e76489ed6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/a8a0971e607d43988abf119e76489ed6 2024-12-10T06:15:07,083 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/c59f4bdb29c24e408fa4bd8c031bdd7f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/c59f4bdb29c24e408fa4bd8c031bdd7f 2024-12-10T06:15:07,083 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/a9192bd362b44a2cb30a649c1be2b18f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/B/a9192bd362b44a2cb30a649c1be2b18f 2024-12-10T06:15:07,087 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/fd36a92445db42bd89b84ab6112a9c20 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/fd36a92445db42bd89b84ab6112a9c20 2024-12-10T06:15:07,087 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/07003007304e49cda3c4a70cce83f1aa to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/07003007304e49cda3c4a70cce83f1aa 2024-12-10T06:15:07,087 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/9d5154847cdc4a8e9ae82c2414ec0366 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/C/9d5154847cdc4a8e9ae82c2414ec0366 2024-12-10T06:15:07,091 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/recovered.edits/564.seqid to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e/recovered.edits/564.seqid 2024-12-10T06:15:07,092 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,092 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T06:15:07,092 DEBUG [PEWorker-1 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T06:15:07,093 DEBUG [PEWorker-1 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T06:15:07,105 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210081c3826acef47159173b39322a12701_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210081c3826acef47159173b39322a12701_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,105 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100e47c99ea01a4d8797a0986f6a20835d_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100e47c99ea01a4d8797a0986f6a20835d_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,105 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121002377bdef5334f179813718ddfa110fb_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121002377bdef5334f179813718ddfa110fb_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,105 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210375aed4b467d48d88b02056d9b37dd57_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210375aed4b467d48d88b02056d9b37dd57_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,106 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103a8da956085848468b26462edf1f8fdd_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103a8da956085848468b26462edf1f8fdd_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,106 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103788d4699ff64a528dea4aee11ed819f_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103788d4699ff64a528dea4aee11ed819f_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,106 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121013536b2525d643cb8096a96fcf2be314_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121013536b2525d643cb8096a96fcf2be314_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,106 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121042ff18ae2e1143f19364598cfd9702fc_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121042ff18ae2e1143f19364598cfd9702fc_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,107 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104869c5d4992248cc8c1ac5c653e0c61f_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104869c5d4992248cc8c1ac5c653e0c61f_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,107 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121069aa1382dbe346c7abd4a947f75fdb62_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121069aa1382dbe346c7abd4a947f75fdb62_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,108 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105a804a12913141bcb3a13916d92a0901_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105a804a12913141bcb3a13916d92a0901_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,108 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210845b2ae6e2ef4672b0238d1cfa8dcda2_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210845b2ae6e2ef4672b0238d1cfa8dcda2_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,108 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121092903f2f96b64500b7538192611dd52b_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121092903f2f96b64500b7538192611dd52b_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,108 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108c566113cc334e1090aab6c72512559b_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108c566113cc334e1090aab6c72512559b_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,108 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109c19614a0245472e8616b9df7a307c1b_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109c19614a0245472e8616b9df7a307c1b_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,109 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a0ffae6bfac142a799c233bccf556d70_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a0ffae6bfac142a799c233bccf556d70_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,110 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b08faf8c0b5e4d8fb680bedfacef9d6e_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b08faf8c0b5e4d8fb680bedfacef9d6e_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,110 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c5e6e3a614a047e2a746aeb58fa47703_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c5e6e3a614a047e2a746aeb58fa47703_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,110 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d249d0ce044846159c8ce0a1f0830929_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d249d0ce044846159c8ce0a1f0830929_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,110 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210edfbfd0021054358a5702be16d52a0b7_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210edfbfd0021054358a5702be16d52a0b7_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,110 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cccbb1fd33a0486e9fa89bc9cade9112_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cccbb1fd33a0486e9fa89bc9cade9112_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,111 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f11fd5b11594409680bc2ff33600eb5a_9a8a17594d375f527e3c6e0f2fcae42e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f11fd5b11594409680bc2ff33600eb5a_9a8a17594d375f527e3c6e0f2fcae42e 2024-12-10T06:15:07,112 DEBUG [PEWorker-1 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T06:15:07,114 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:07,117 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T06:15:07,120 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T06:15:07,121 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:07,121 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T06:15:07,122 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733811307121"}]},"ts":"9223372036854775807"} 2024-12-10T06:15:07,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T06:15:07,124 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9a8a17594d375f527e3c6e0f2fcae42e, NAME => 'TestAcidGuarantees,,1733811279514.9a8a17594d375f527e3c6e0f2fcae42e.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T06:15:07,124 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T06:15:07,124 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733811307124"}]},"ts":"9223372036854775807"} 2024-12-10T06:15:07,126 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T06:15:07,127 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:07,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-12-10T06:15:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T06:15:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T06:15:07,179 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:07,187 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=258 (was 255) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_686407029_22 at /127.0.0.1:42526 [Waiting for operation #471] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1197328417_22 at /127.0.0.1:42492 [Waiting for operation #497] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_686407029_22 at /127.0.0.1:45222 [Waiting for operation #818] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1197328417_22 at /127.0.0.1:42536 [Waiting for operation #474] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=473 (was 422) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7291 (was 7157) - AvailableMemoryMB LEAK? - 2024-12-10T06:15:07,196 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=258, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=473, ProcessCount=11, AvailableMemoryMB=7291 2024-12-10T06:15:07,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T06:15:07,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:15:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:07,199 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T06:15:07,199 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:07,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-12-10T06:15:07,200 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T06:15:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T06:15:07,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742063_1239 (size=1061) 2024-12-10T06:15:07,280 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T06:15:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T06:15:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T06:15:07,608 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6 2024-12-10T06:15:07,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742064_1240 (size=53) 2024-12-10T06:15:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T06:15:08,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:15:08,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 062b93d61f64862ee02e4bde4cb11903, disabling compactions & flushes 2024-12-10T06:15:08,016 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:08,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:08,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. after waiting 0 ms 2024-12-10T06:15:08,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:08,016 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:08,016 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 062b93d61f64862ee02e4bde4cb11903: Waiting for close lock at 1733811308016Disabling compacts and flushes for region at 1733811308016Disabling writes for close at 1733811308016Writing region close event to WAL at 1733811308016Closed at 1733811308016 2024-12-10T06:15:08,018 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T06:15:08,018 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733811308018"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733811308018"}]},"ts":"1733811308018"} 2024-12-10T06:15:08,020 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T06:15:08,021 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T06:15:08,022 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811308021"}]},"ts":"1733811308021"} 2024-12-10T06:15:08,032 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T06:15:08,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=062b93d61f64862ee02e4bde4cb11903, ASSIGN}] 2024-12-10T06:15:08,033 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=062b93d61f64862ee02e4bde4cb11903, ASSIGN 2024-12-10T06:15:08,034 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=062b93d61f64862ee02e4bde4cb11903, ASSIGN; state=OFFLINE, location=c42c682b34f7,43983,1733811247584; forceNewPlan=false, retain=false 2024-12-10T06:15:08,185 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=062b93d61f64862ee02e4bde4cb11903, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:08,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=062b93d61f64862ee02e4bde4cb11903, ASSIGN because future has completed 2024-12-10T06:15:08,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:15:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T06:15:08,342 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:08,342 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:15:08,342 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,343 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:15:08,343 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,343 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,344 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,346 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:15:08,346 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 062b93d61f64862ee02e4bde4cb11903 columnFamilyName A 2024-12-10T06:15:08,346 DEBUG [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:08,346 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] regionserver.HStore(327): Store=062b93d61f64862ee02e4bde4cb11903/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:15:08,347 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,348 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:15:08,348 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 062b93d61f64862ee02e4bde4cb11903 columnFamilyName B 2024-12-10T06:15:08,348 DEBUG [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:08,348 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] regionserver.HStore(327): Store=062b93d61f64862ee02e4bde4cb11903/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:15:08,348 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,349 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:15:08,349 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 062b93d61f64862ee02e4bde4cb11903 columnFamilyName C 2024-12-10T06:15:08,350 DEBUG [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:08,351 INFO [StoreOpener-062b93d61f64862ee02e4bde4cb11903-1 {}] regionserver.HStore(327): Store=062b93d61f64862ee02e4bde4cb11903/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:15:08,351 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:08,351 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,352 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,352 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,353 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,353 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,354 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:15:08,355 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,357 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:15:08,357 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 062b93d61f64862ee02e4bde4cb11903; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62330072, jitterRate=-0.07120954990386963}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:15:08,358 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:08,358 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 062b93d61f64862ee02e4bde4cb11903: Running coprocessor pre-open hook at 1733811308343Writing region info on filesystem at 1733811308343Initializing all the Stores at 1733811308344 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811308344Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811308344Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811308344Cleaning up temporary data from old regions at 1733811308353 (+9 ms)Running coprocessor post-open hooks at 1733811308358 (+5 ms)Region opened successfully at 1733811308358 2024-12-10T06:15:08,359 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., pid=57, masterSystemTime=1733811308339 2024-12-10T06:15:08,361 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:08,361 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:08,362 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=062b93d61f64862ee02e4bde4cb11903, regionState=OPEN, openSeqNum=2, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:08,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:15:08,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-10T06:15:08,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 in 177 msec 2024-12-10T06:15:08,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-10T06:15:08,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=062b93d61f64862ee02e4bde4cb11903, ASSIGN in 333 msec 2024-12-10T06:15:08,368 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T06:15:08,369 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811308368"}]},"ts":"1733811308368"} 2024-12-10T06:15:08,370 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T06:15:08,371 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T06:15:08,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-12-10T06:15:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T06:15:09,355 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:09,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25146acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,358 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af95eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,359 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405509c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,360 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,363 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T06:15:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c42c682b34f7,37415,1733811246834 2024-12-10T06:15:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c4a80f0 2024-12-10T06:15:09,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T06:15:09,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T06:15:09,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,366 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b917ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,368 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192ddb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,369 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3130a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,370 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121adf72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,372 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ef5b5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,373 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a9464e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,375 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7ac7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,376 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135894d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,378 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,378 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,378 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a2249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,380 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b8cfae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,382 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e45aa9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,384 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a25cab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,386 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26195fe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,387 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ebeca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,392 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,393 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d509c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,394 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344ac1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,400 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3248a5f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,402 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bec7c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5621a0fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:09,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:09,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c477195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:09,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:09,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:09,408 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e49b997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:09,412 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,413 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,424 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:09,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T06:15:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,430 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,431 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,432 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-10T06:15:09,435 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T06:15:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,436 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:09,437 DEBUG [Thread-1128 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,437 DEBUG [Thread-1128 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,449 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,465 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/c824db809fd245be9c07c80aafe5f9af is 50, key is test_row_0/A:col10/1733811309418/Put/seqid=0 2024-12-10T06:15:09,477 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,491 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:09,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:09,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:09,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:09,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742065_1241 (size=12001) 2024-12-10T06:15:09,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:15:09,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/c824db809fd245be9c07c80aafe5f9af 2024-12-10T06:15:09,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811369502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811369504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811369504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811309474, max timestamp=1733811309508Num uniques 859; 2024-12-10T06:15:09,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811309474, max timestamp=1733811309508Num uniques 859; 2024-12-10T06:15:09,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:15:09,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811309474, max timestamp=1733811309508Num uniques 859; 2024-12-10T06:15:09,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:15:09,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:15:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T06:15:09,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/062a5762a75b4d49b11faa969959a840 is 50, key is test_row_0/B:col10/1733811309418/Put/seqid=0 2024-12-10T06:15:09,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742066_1242 (size=12001) 2024-12-10T06:15:09,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/062a5762a75b4d49b11faa969959a840 2024-12-10T06:15:09,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811369583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811369583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T06:15:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:09,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:09,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:09,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:09,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:09,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/3c1420b027684ead8541f6d537d2b9ef is 50, key is test_row_0/C:col10/1733811309418/Put/seqid=0 2024-12-10T06:15:09,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811369616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811369616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811369616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742067_1243 (size=12001) 2024-12-10T06:15:09,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/3c1420b027684ead8541f6d537d2b9ef 2024-12-10T06:15:09,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/c824db809fd245be9c07c80aafe5f9af as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c824db809fd245be9c07c80aafe5f9af 2024-12-10T06:15:09,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c824db809fd245be9c07c80aafe5f9af, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T06:15:09,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/062a5762a75b4d49b11faa969959a840 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/062a5762a75b4d49b11faa969959a840 2024-12-10T06:15:09,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/062a5762a75b4d49b11faa969959a840, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T06:15:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/3c1420b027684ead8541f6d537d2b9ef as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/3c1420b027684ead8541f6d537d2b9ef 2024-12-10T06:15:09,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/3c1420b027684ead8541f6d537d2b9ef, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T06:15:09,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 062b93d61f64862ee02e4bde4cb11903 in 270ms, sequenceid=12, compaction requested=false 2024-12-10T06:15:09,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:09,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811309568, max timestamp=1733811309702Num uniques 1018; 2024-12-10T06:15:09,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T06:15:09,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7685, totalHeapSize=17712, min timestamp=1733811309568, max timestamp=1733811309703Num uniques 1018; 2024-12-10T06:15:09,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46615, new segment data size=7685, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T06:15:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:09,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811309568, max timestamp=1733811309703Num uniques 1018; 2024-12-10T06:15:09,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T06:15:09,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=33.54 KB heapSize=85.31 KB 2024-12-10T06:15:09,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:09,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/e7198e362871452ca0a083c72b54f8b0 is 50, key is test_row_0/A:col10/1733811309702/Put/seqid=0 2024-12-10T06:15:09,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T06:15:09,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:09,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:09,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:09,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:09,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T06:15:09,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742068_1244 (size=12001) 2024-12-10T06:15:09,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/e7198e362871452ca0a083c72b54f8b0 2024-12-10T06:15:09,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/a280a2ce5a674ea3a5e381517a27363f is 50, key is test_row_0/B:col10/1733811309702/Put/seqid=0 2024-12-10T06:15:09,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742069_1245 (size=12001) 2024-12-10T06:15:09,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/a280a2ce5a674ea3a5e381517a27363f 2024-12-10T06:15:09,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c20ec0bbd1494329a35897a271fdc74e is 50, key is test_row_0/C:col10/1733811309702/Put/seqid=0 2024-12-10T06:15:09,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1733811309766, max timestamp=1733811309843Num uniques 859; 2024-12-10T06:15:09,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:09,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733811309766, max timestamp=1733811309843Num uniques 859; 2024-12-10T06:15:09,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:09,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733811309766, max timestamp=1733811309843Num uniques 859; 2024-12-10T06:15:09,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T06:15:09,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742070_1246 (size=12001) 2024-12-10T06:15:09,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c20ec0bbd1494329a35897a271fdc74e 2024-12-10T06:15:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/e7198e362871452ca0a083c72b54f8b0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/e7198e362871452ca0a083c72b54f8b0 2024-12-10T06:15:09,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T06:15:09,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:09,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:09,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:09,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:09,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:09,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811369890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733811309883, max timestamp=1733811309894Num uniques 1068; 2024-12-10T06:15:09,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733811309883, max timestamp=1733811309894Num uniques 1018; 2024-12-10T06:15:09,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T06:15:09,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733811309883, max timestamp=1733811309894Num uniques 1068; 2024-12-10T06:15:09,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-10T06:15:09,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/e7198e362871452ca0a083c72b54f8b0, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T06:15:09,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-10T06:15:09,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/a280a2ce5a674ea3a5e381517a27363f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a280a2ce5a674ea3a5e381517a27363f 2024-12-10T06:15:09,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a280a2ce5a674ea3a5e381517a27363f, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T06:15:09,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c20ec0bbd1494329a35897a271fdc74e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c20ec0bbd1494329a35897a271fdc74e 2024-12-10T06:15:09,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c20ec0bbd1494329a35897a271fdc74e, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T06:15:09,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=93.93 KB/96180 for 062b93d61f64862ee02e4bde4cb11903 in 221ms, sequenceid=51, compaction requested=false 2024-12-10T06:15:09,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:09,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=107.34 KB heapSize=299.77 KB 2024-12-10T06:15:09,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:09,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:09,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:09,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:09,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:09,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:09,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7da0c999bdd348deab3cc0fd5e3083d1 is 50, key is test_row_0/A:col10/1733811309935/Put/seqid=0 2024-12-10T06:15:09,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811369966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811369964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811369974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811369975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:09,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742071_1247 (size=14341) 2024-12-10T06:15:09,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7da0c999bdd348deab3cc0fd5e3083d1 2024-12-10T06:15:10,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811370005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/d4997c2ff6c74eff8f46ff7df476e6ab is 50, key is test_row_0/B:col10/1733811309935/Put/seqid=0 2024-12-10T06:15:10,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742072_1248 (size=12001) 2024-12-10T06:15:10,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/d4997c2ff6c74eff8f46ff7df476e6ab 2024-12-10T06:15:10,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/82ed9802f3b141b48b9d4561125dd8c4 is 50, key is test_row_0/C:col10/1733811309935/Put/seqid=0 2024-12-10T06:15:10,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742073_1249 (size=12001) 2024-12-10T06:15:10,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T06:15:10,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:10,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:10,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T06:15:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811370085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811370086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811370086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811370086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T06:15:10,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:10,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:10,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:10,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811370215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811370295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811370295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811370296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811370296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T06:15:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:10,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:10,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/82ed9802f3b141b48b9d4561125dd8c4 2024-12-10T06:15:10,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7da0c999bdd348deab3cc0fd5e3083d1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7da0c999bdd348deab3cc0fd5e3083d1 2024-12-10T06:15:10,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7da0c999bdd348deab3cc0fd5e3083d1, entries=200, sequenceid=102, filesize=14.0 K 2024-12-10T06:15:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/d4997c2ff6c74eff8f46ff7df476e6ab as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/d4997c2ff6c74eff8f46ff7df476e6ab 2024-12-10T06:15:10,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/d4997c2ff6c74eff8f46ff7df476e6ab, entries=150, sequenceid=102, filesize=11.7 K 2024-12-10T06:15:10,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/82ed9802f3b141b48b9d4561125dd8c4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/82ed9802f3b141b48b9d4561125dd8c4 2024-12-10T06:15:10,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/82ed9802f3b141b48b9d4561125dd8c4, entries=150, sequenceid=102, filesize=11.7 K 2024-12-10T06:15:10,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~339.56 KB/347712, currentSize=67.09 KB/68700 for 062b93d61f64862ee02e4bde4cb11903 in 527ms, sequenceid=102, compaction requested=true 2024-12-10T06:15:10,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:10,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:10,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:10,464 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:10,464 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:10,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:10,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:10,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:10,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:10,465 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:10,465 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/B is initiating minor compaction (all files) 2024-12-10T06:15:10,465 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/B in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,465 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/062a5762a75b4d49b11faa969959a840, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a280a2ce5a674ea3a5e381517a27363f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/d4997c2ff6c74eff8f46ff7df476e6ab] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=35.2 K 2024-12-10T06:15:10,466 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:10,466 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/A is initiating minor compaction (all files) 2024-12-10T06:15:10,466 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/A in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,466 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c824db809fd245be9c07c80aafe5f9af, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/e7198e362871452ca0a083c72b54f8b0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7da0c999bdd348deab3cc0fd5e3083d1] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=37.4 K 2024-12-10T06:15:10,466 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c824db809fd245be9c07c80aafe5f9af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733811309418 2024-12-10T06:15:10,466 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 062a5762a75b4d49b11faa969959a840, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733811309418 2024-12-10T06:15:10,467 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7198e362871452ca0a083c72b54f8b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811309581 2024-12-10T06:15:10,467 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting a280a2ce5a674ea3a5e381517a27363f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811309581 2024-12-10T06:15:10,468 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d4997c2ff6c74eff8f46ff7df476e6ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733811309933 2024-12-10T06:15:10,468 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7da0c999bdd348deab3cc0fd5e3083d1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733811309924 2024-12-10T06:15:10,487 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:10,488 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/e3c8a46525b24de08b3ec41325ae3a84 is 50, key is test_row_0/B:col10/1733811309935/Put/seqid=0 2024-12-10T06:15:10,501 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#A#compaction#202 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:10,503 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7458f389d76a4370867456119d300b0b is 50, key is test_row_0/A:col10/1733811309935/Put/seqid=0 2024-12-10T06:15:10,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T06:15:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,518 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T06:15:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:10,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:10,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742074_1250 (size=12104) 2024-12-10T06:15:10,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742075_1251 (size=12104) 2024-12-10T06:15:10,558 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/e3c8a46525b24de08b3ec41325ae3a84 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/e3c8a46525b24de08b3ec41325ae3a84 2024-12-10T06:15:10,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/bc93688ced2e4061b0a636baad186a01 is 50, key is test_row_0/A:col10/1733811309973/Put/seqid=0 2024-12-10T06:15:10,563 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7458f389d76a4370867456119d300b0b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7458f389d76a4370867456119d300b0b 2024-12-10T06:15:10,566 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/B of 062b93d61f64862ee02e4bde4cb11903 into e3c8a46525b24de08b3ec41325ae3a84(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:10,566 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:10,566 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/B, priority=13, startTime=1733811310464; duration=0sec 2024-12-10T06:15:10,566 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:10,566 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:B 2024-12-10T06:15:10,566 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:10,568 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:10,568 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/C is initiating minor compaction (all files) 2024-12-10T06:15:10,568 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/C in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:10,568 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/3c1420b027684ead8541f6d537d2b9ef, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c20ec0bbd1494329a35897a271fdc74e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/82ed9802f3b141b48b9d4561125dd8c4] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=35.2 K 2024-12-10T06:15:10,570 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c1420b027684ead8541f6d537d2b9ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733811309418 2024-12-10T06:15:10,571 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting c20ec0bbd1494329a35897a271fdc74e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811309581 2024-12-10T06:15:10,571 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/A of 062b93d61f64862ee02e4bde4cb11903 into 7458f389d76a4370867456119d300b0b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:10,571 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:10,571 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/A, priority=13, startTime=1733811310463; duration=0sec 2024-12-10T06:15:10,571 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ed9802f3b141b48b9d4561125dd8c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733811309933 2024-12-10T06:15:10,572 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:10,572 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:A 2024-12-10T06:15:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T06:15:10,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742076_1252 (size=12001) 2024-12-10T06:15:10,609 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#C#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:10,609 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/117ca9c04c9e44bca1aa712818257e24 is 50, key is test_row_0/C:col10/1733811309935/Put/seqid=0 2024-12-10T06:15:10,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742077_1253 (size=12104) 2024-12-10T06:15:10,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811370614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811370615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811370616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811370618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811370621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,628 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/117ca9c04c9e44bca1aa712818257e24 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/117ca9c04c9e44bca1aa712818257e24 2024-12-10T06:15:10,633 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/C of 062b93d61f64862ee02e4bde4cb11903 into 117ca9c04c9e44bca1aa712818257e24(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:10,633 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:10,633 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/C, priority=13, startTime=1733811310464; duration=0sec 2024-12-10T06:15:10,633 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:10,633 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:C 2024-12-10T06:15:10,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811370725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811370725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811370735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811370735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811370735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811370935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811370935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811370945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811370946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811370946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,005 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/bc93688ced2e4061b0a636baad186a01 2024-12-10T06:15:11,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/340d182b51e04c8d872cd3572659bde8 is 50, key is test_row_0/B:col10/1733811309973/Put/seqid=0 2024-12-10T06:15:11,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742078_1254 (size=12001) 2024-12-10T06:15:11,118 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:15:11,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811371246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811371246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811371255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811371256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811371256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,426 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/340d182b51e04c8d872cd3572659bde8 2024-12-10T06:15:11,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/60c855ab04ad4898b506a317a7278ee0 is 50, key is test_row_0/C:col10/1733811309973/Put/seqid=0 2024-12-10T06:15:11,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742079_1255 (size=12001) 2024-12-10T06:15:11,499 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/60c855ab04ad4898b506a317a7278ee0 2024-12-10T06:15:11,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/bc93688ced2e4061b0a636baad186a01 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bc93688ced2e4061b0a636baad186a01 2024-12-10T06:15:11,517 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bc93688ced2e4061b0a636baad186a01, entries=150, sequenceid=115, filesize=11.7 K 2024-12-10T06:15:11,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/340d182b51e04c8d872cd3572659bde8 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/340d182b51e04c8d872cd3572659bde8 2024-12-10T06:15:11,523 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/340d182b51e04c8d872cd3572659bde8, entries=150, sequenceid=115, filesize=11.7 K 2024-12-10T06:15:11,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/60c855ab04ad4898b506a317a7278ee0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/60c855ab04ad4898b506a317a7278ee0 2024-12-10T06:15:11,536 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/60c855ab04ad4898b506a317a7278ee0, entries=150, sequenceid=115, filesize=11.7 K 2024-12-10T06:15:11,538 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 062b93d61f64862ee02e4bde4cb11903 in 1019ms, sequenceid=115, compaction requested=false 2024-12-10T06:15:11,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:11,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:11,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-10T06:15:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-10T06:15:11,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-10T06:15:11,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1040 sec 2024-12-10T06:15:11,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.1110 sec 2024-12-10T06:15:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T06:15:11,585 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:11,586 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-10T06:15:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T06:15:11,589 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:11,590 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:11,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T06:15:11,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T06:15:11,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:11,744 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T06:15:11,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:11,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:11,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:11,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:11,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:11,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:11,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/a20bf6c5808949018e3b80add166af03 is 50, key is test_row_0/A:col10/1733811310615/Put/seqid=0 2024-12-10T06:15:11,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:11,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742080_1256 (size=12151) 2024-12-10T06:15:11,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811371778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811371778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,785 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/a20bf6c5808949018e3b80add166af03 2024-12-10T06:15:11,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811371780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811371782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811371783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/fde44ca76b8e44c7ac73ebc5fd3a849d is 50, key is test_row_0/B:col10/1733811310615/Put/seqid=0 2024-12-10T06:15:11,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742081_1257 (size=12151) 2024-12-10T06:15:11,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/fde44ca76b8e44c7ac73ebc5fd3a849d 2024-12-10T06:15:11,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/0583ede398bf47fc850c7c23befb44e1 is 50, key is test_row_0/C:col10/1733811310615/Put/seqid=0 2024-12-10T06:15:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742082_1258 (size=12151) 2024-12-10T06:15:11,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811371895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811371895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811371895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811371895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811371895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T06:15:12,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811372108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811372109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811372113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811372115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811372115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T06:15:12,279 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/0583ede398bf47fc850c7c23befb44e1 2024-12-10T06:15:12,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/a20bf6c5808949018e3b80add166af03 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a20bf6c5808949018e3b80add166af03 2024-12-10T06:15:12,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a20bf6c5808949018e3b80add166af03, entries=150, sequenceid=138, filesize=11.9 K 2024-12-10T06:15:12,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/fde44ca76b8e44c7ac73ebc5fd3a849d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/fde44ca76b8e44c7ac73ebc5fd3a849d 2024-12-10T06:15:12,300 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/fde44ca76b8e44c7ac73ebc5fd3a849d, entries=150, sequenceid=138, filesize=11.9 K 2024-12-10T06:15:12,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/0583ede398bf47fc850c7c23befb44e1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0583ede398bf47fc850c7c23befb44e1 2024-12-10T06:15:12,308 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0583ede398bf47fc850c7c23befb44e1, entries=150, sequenceid=138, filesize=11.9 K 2024-12-10T06:15:12,309 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 062b93d61f64862ee02e4bde4cb11903 in 566ms, sequenceid=138, compaction requested=true 2024-12-10T06:15:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-10T06:15:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-10T06:15:12,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-10T06:15:12,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 722 msec 2024-12-10T06:15:12,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 729 msec 2024-12-10T06:15:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:12,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T06:15:12,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:12,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:12,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:12,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:12,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:12,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:12,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/24f9d7704b98498984ec426b89ce23f1 is 50, key is test_row_0/A:col10/1733811312427/Put/seqid=0 2024-12-10T06:15:12,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742083_1259 (size=12151) 2024-12-10T06:15:12,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811372448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811372449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811372450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811372451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811372453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811372555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811372566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811372566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811372566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811372566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T06:15:12,726 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:12,726 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-10T06:15:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T06:15:12,729 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:12,730 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:12,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:12,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811372765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811372775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811372775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811372775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811372775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T06:15:12,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/24f9d7704b98498984ec426b89ce23f1 2024-12-10T06:15:12,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/f9f4528815194c99b28e7e61c455506c is 50, key is test_row_0/B:col10/1733811312427/Put/seqid=0 2024-12-10T06:15:12,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742084_1260 (size=12151) 2024-12-10T06:15:12,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/f9f4528815194c99b28e7e61c455506c 2024-12-10T06:15:12,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T06:15:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:12,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:12,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/0b22f652dd2e44a69b767e3754ed343d is 50, key is test_row_0/C:col10/1733811312427/Put/seqid=0 2024-12-10T06:15:12,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742085_1261 (size=12151) 2024-12-10T06:15:13,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T06:15:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:13,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:13,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:13,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:13,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T06:15:13,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811373075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811373085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811373086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811373086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811373086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T06:15:13,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:13,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:13,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:13,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:13,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:13,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/0b22f652dd2e44a69b767e3754ed343d 2024-12-10T06:15:13,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/24f9d7704b98498984ec426b89ce23f1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/24f9d7704b98498984ec426b89ce23f1 2024-12-10T06:15:13,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/24f9d7704b98498984ec426b89ce23f1, entries=150, sequenceid=152, filesize=11.9 K 2024-12-10T06:15:13,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/f9f4528815194c99b28e7e61c455506c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/f9f4528815194c99b28e7e61c455506c 2024-12-10T06:15:13,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/f9f4528815194c99b28e7e61c455506c, entries=150, sequenceid=152, filesize=11.9 K 2024-12-10T06:15:13,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/0b22f652dd2e44a69b767e3754ed343d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0b22f652dd2e44a69b767e3754ed343d 2024-12-10T06:15:13,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0b22f652dd2e44a69b767e3754ed343d, entries=150, sequenceid=152, filesize=11.9 K 2024-12-10T06:15:13,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 062b93d61f64862ee02e4bde4cb11903 in 897ms, sequenceid=152, compaction requested=true 2024-12-10T06:15:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:13,325 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:13,325 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:13,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:13,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:13,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:13,327 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:13,327 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:13,327 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/A is initiating minor compaction (all files) 2024-12-10T06:15:13,327 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/B is initiating minor compaction (all files) 2024-12-10T06:15:13,327 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/B in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:13,327 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/A in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:13,327 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7458f389d76a4370867456119d300b0b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bc93688ced2e4061b0a636baad186a01, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a20bf6c5808949018e3b80add166af03, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/24f9d7704b98498984ec426b89ce23f1] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=47.3 K 2024-12-10T06:15:13,327 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/e3c8a46525b24de08b3ec41325ae3a84, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/340d182b51e04c8d872cd3572659bde8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/fde44ca76b8e44c7ac73ebc5fd3a849d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/f9f4528815194c99b28e7e61c455506c] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=47.3 K 2024-12-10T06:15:13,328 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting e3c8a46525b24de08b3ec41325ae3a84, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733811309933 2024-12-10T06:15:13,328 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7458f389d76a4370867456119d300b0b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733811309933 2024-12-10T06:15:13,329 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 340d182b51e04c8d872cd3572659bde8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733811309953 2024-12-10T06:15:13,329 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc93688ced2e4061b0a636baad186a01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733811309953 2024-12-10T06:15:13,330 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting fde44ca76b8e44c7ac73ebc5fd3a849d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733811310611 2024-12-10T06:15:13,330 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting a20bf6c5808949018e3b80add166af03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733811310611 2024-12-10T06:15:13,330 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting f9f4528815194c99b28e7e61c455506c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733811311776 2024-12-10T06:15:13,330 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24f9d7704b98498984ec426b89ce23f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733811311776 2024-12-10T06:15:13,343 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:13,344 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/ccc1b289fdac4c8bab8d4b17d2235421 is 50, key is test_row_0/A:col10/1733811312427/Put/seqid=0 2024-12-10T06:15:13,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T06:15:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:13,347 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:13,347 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T06:15:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:13,347 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/5d84f7e638a1470a8b3ad27a5e914c01 is 50, key is test_row_0/B:col10/1733811312427/Put/seqid=0 2024-12-10T06:15:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T06:15:13,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/dfbb475fb1c64dac8db841242d5e98af is 50, key is test_row_0/A:col10/1733811312438/Put/seqid=0 2024-12-10T06:15:13,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742086_1262 (size=12391) 2024-12-10T06:15:13,390 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/ccc1b289fdac4c8bab8d4b17d2235421 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ccc1b289fdac4c8bab8d4b17d2235421 2024-12-10T06:15:13,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742087_1263 (size=12391) 2024-12-10T06:15:13,402 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/A of 062b93d61f64862ee02e4bde4cb11903 into ccc1b289fdac4c8bab8d4b17d2235421(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:13,402 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:13,402 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/A, priority=12, startTime=1733811313325; duration=0sec 2024-12-10T06:15:13,402 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:13,402 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:A 2024-12-10T06:15:13,402 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:13,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742088_1264 (size=12151) 2024-12-10T06:15:13,406 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:13,406 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/C is initiating minor compaction (all files) 2024-12-10T06:15:13,406 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/C in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:13,406 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/117ca9c04c9e44bca1aa712818257e24, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/60c855ab04ad4898b506a317a7278ee0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0583ede398bf47fc850c7c23befb44e1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0b22f652dd2e44a69b767e3754ed343d] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=47.3 K 2024-12-10T06:15:13,406 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 117ca9c04c9e44bca1aa712818257e24, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733811309933 2024-12-10T06:15:13,410 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c855ab04ad4898b506a317a7278ee0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733811309953 2024-12-10T06:15:13,411 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0583ede398bf47fc850c7c23befb44e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733811310611 2024-12-10T06:15:13,412 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b22f652dd2e44a69b767e3754ed343d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733811311776 2024-12-10T06:15:13,437 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#C#compaction#216 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:13,437 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/f164b9b2da1243c2abfdd8a31a1c94f4 is 50, key is test_row_0/C:col10/1733811312427/Put/seqid=0 2024-12-10T06:15:13,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742089_1265 (size=12391) 2024-12-10T06:15:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:13,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:13,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811373612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811373613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811373619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811373619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811373619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811373725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811373726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811373735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811373735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811373735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,804 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/5d84f7e638a1470a8b3ad27a5e914c01 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/5d84f7e638a1470a8b3ad27a5e914c01 2024-12-10T06:15:13,805 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/dfbb475fb1c64dac8db841242d5e98af 2024-12-10T06:15:13,810 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/B of 062b93d61f64862ee02e4bde4cb11903 into 5d84f7e638a1470a8b3ad27a5e914c01(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:13,810 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:13,810 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/B, priority=12, startTime=1733811313325; duration=0sec 2024-12-10T06:15:13,810 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:13,810 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:B 2024-12-10T06:15:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/4fa223826cc34dba806ca1c3ad12e40f is 50, key is test_row_0/B:col10/1733811312438/Put/seqid=0 2024-12-10T06:15:13,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742090_1266 (size=12151) 2024-12-10T06:15:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T06:15:13,874 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/f164b9b2da1243c2abfdd8a31a1c94f4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/f164b9b2da1243c2abfdd8a31a1c94f4 2024-12-10T06:15:13,879 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/C of 062b93d61f64862ee02e4bde4cb11903 into f164b9b2da1243c2abfdd8a31a1c94f4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:13,879 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:13,879 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/C, priority=12, startTime=1733811313326; duration=0sec 2024-12-10T06:15:13,879 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:13,879 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:C 2024-12-10T06:15:13,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811373935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811373936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811373945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811373945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811373945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/4fa223826cc34dba806ca1c3ad12e40f 2024-12-10T06:15:14,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811374245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811374246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811374255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811374255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811374255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/48547ec1758c4bffb2ee17f42f134b3d is 50, key is test_row_0/C:col10/1733811312438/Put/seqid=0 2024-12-10T06:15:14,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742091_1267 (size=12151) 2024-12-10T06:15:14,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/48547ec1758c4bffb2ee17f42f134b3d 2024-12-10T06:15:14,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/dfbb475fb1c64dac8db841242d5e98af as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/dfbb475fb1c64dac8db841242d5e98af 2024-12-10T06:15:14,687 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/dfbb475fb1c64dac8db841242d5e98af, entries=150, sequenceid=170, filesize=11.9 K 2024-12-10T06:15:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/4fa223826cc34dba806ca1c3ad12e40f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/4fa223826cc34dba806ca1c3ad12e40f 2024-12-10T06:15:14,693 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/4fa223826cc34dba806ca1c3ad12e40f, entries=150, sequenceid=170, filesize=11.9 K 2024-12-10T06:15:14,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/48547ec1758c4bffb2ee17f42f134b3d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/48547ec1758c4bffb2ee17f42f134b3d 2024-12-10T06:15:14,698 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/48547ec1758c4bffb2ee17f42f134b3d, entries=150, sequenceid=170, filesize=11.9 K 2024-12-10T06:15:14,699 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 062b93d61f64862ee02e4bde4cb11903 in 1352ms, sequenceid=170, compaction requested=false 2024-12-10T06:15:14,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:14,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-10T06:15:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-10T06:15:14,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-10T06:15:14,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9720 sec 2024-12-10T06:15:14,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.9790 sec 2024-12-10T06:15:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:14,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:15:14,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:14,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:14,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:14,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:14,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:14,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:14,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/73d12e41c49d474aa9b31e0031e08eb4 is 50, key is test_row_0/A:col10/1733811314759/Put/seqid=0 2024-12-10T06:15:14,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742092_1268 (size=14541) 2024-12-10T06:15:14,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/73d12e41c49d474aa9b31e0031e08eb4 2024-12-10T06:15:14,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811374778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811374780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811374781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811374782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811374783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/726f0eeb1c2f4e4bbc3707003a5af1b5 is 50, key is test_row_0/B:col10/1733811314759/Put/seqid=0 2024-12-10T06:15:14,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742093_1269 (size=12151) 2024-12-10T06:15:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T06:15:14,876 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:14,876 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-10T06:15:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T06:15:14,879 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:14,879 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:14,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:14,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811374895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811374896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811374896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811374896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811374896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T06:15:15,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T06:15:15,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:15,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811375105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811375105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811375105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811375105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811375105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T06:15:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T06:15:15,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/726f0eeb1c2f4e4bbc3707003a5af1b5 2024-12-10T06:15:15,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/a1ea4956df4847b6bf1c6597206fbdec is 50, key is test_row_0/C:col10/1733811314759/Put/seqid=0 2024-12-10T06:15:15,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742094_1270 (size=12151) 2024-12-10T06:15:15,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T06:15:15,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811375415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811375415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811375416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811375416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811375416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T06:15:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T06:15:15,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/a1ea4956df4847b6bf1c6597206fbdec 2024-12-10T06:15:15,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/73d12e41c49d474aa9b31e0031e08eb4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/73d12e41c49d474aa9b31e0031e08eb4 2024-12-10T06:15:15,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/73d12e41c49d474aa9b31e0031e08eb4, entries=200, sequenceid=190, filesize=14.2 K 2024-12-10T06:15:15,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/726f0eeb1c2f4e4bbc3707003a5af1b5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/726f0eeb1c2f4e4bbc3707003a5af1b5 2024-12-10T06:15:15,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T06:15:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:15,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/726f0eeb1c2f4e4bbc3707003a5af1b5, entries=150, sequenceid=190, filesize=11.9 K 2024-12-10T06:15:15,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/a1ea4956df4847b6bf1c6597206fbdec as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a1ea4956df4847b6bf1c6597206fbdec 2024-12-10T06:15:15,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a1ea4956df4847b6bf1c6597206fbdec, entries=150, sequenceid=190, filesize=11.9 K 2024-12-10T06:15:15,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 062b93d61f64862ee02e4bde4cb11903 in 902ms, sequenceid=190, compaction requested=true 2024-12-10T06:15:15,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:15,661 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:15,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:15,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:15,662 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:15,663 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:15,663 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/A is initiating minor compaction (all files) 2024-12-10T06:15:15,663 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/A in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,663 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ccc1b289fdac4c8bab8d4b17d2235421, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/dfbb475fb1c64dac8db841242d5e98af, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/73d12e41c49d474aa9b31e0031e08eb4] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=38.2 K 2024-12-10T06:15:15,664 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:15,664 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/B is initiating minor compaction (all files) 2024-12-10T06:15:15,664 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/B in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,664 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/5d84f7e638a1470a8b3ad27a5e914c01, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/4fa223826cc34dba806ca1c3ad12e40f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/726f0eeb1c2f4e4bbc3707003a5af1b5] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=35.8 K 2024-12-10T06:15:15,664 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc1b289fdac4c8bab8d4b17d2235421, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733811311776 2024-12-10T06:15:15,665 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d84f7e638a1470a8b3ad27a5e914c01, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733811311776 2024-12-10T06:15:15,665 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfbb475fb1c64dac8db841242d5e98af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733811312438 2024-12-10T06:15:15,666 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fa223826cc34dba806ca1c3ad12e40f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733811312438 2024-12-10T06:15:15,666 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73d12e41c49d474aa9b31e0031e08eb4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811313616 2024-12-10T06:15:15,666 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 726f0eeb1c2f4e4bbc3707003a5af1b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811313616 2024-12-10T06:15:15,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:15,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:15,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:15,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:15,688 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:15,689 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/c91951409c5846348a394789495934e0 is 50, key is test_row_0/A:col10/1733811314759/Put/seqid=0 2024-12-10T06:15:15,692 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:15,693 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/bb0e250b4af045a0b0c57a5f0fc19c24 is 50, key is test_row_0/B:col10/1733811314759/Put/seqid=0 2024-12-10T06:15:15,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742095_1271 (size=12493) 2024-12-10T06:15:15,707 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/c91951409c5846348a394789495934e0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c91951409c5846348a394789495934e0 2024-12-10T06:15:15,714 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/A of 062b93d61f64862ee02e4bde4cb11903 into c91951409c5846348a394789495934e0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:15,715 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:15,715 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/A, priority=13, startTime=1733811315661; duration=0sec 2024-12-10T06:15:15,715 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:15,715 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:A 2024-12-10T06:15:15,715 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:15,717 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:15,717 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/C is initiating minor compaction (all files) 2024-12-10T06:15:15,717 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/C in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,718 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/f164b9b2da1243c2abfdd8a31a1c94f4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/48547ec1758c4bffb2ee17f42f134b3d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a1ea4956df4847b6bf1c6597206fbdec] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=35.8 K 2024-12-10T06:15:15,718 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f164b9b2da1243c2abfdd8a31a1c94f4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733811311776 2024-12-10T06:15:15,718 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48547ec1758c4bffb2ee17f42f134b3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733811312438 2024-12-10T06:15:15,719 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ea4956df4847b6bf1c6597206fbdec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811313616 2024-12-10T06:15:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742096_1272 (size=12493) 2024-12-10T06:15:15,737 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/bb0e250b4af045a0b0c57a5f0fc19c24 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/bb0e250b4af045a0b0c57a5f0fc19c24 2024-12-10T06:15:15,749 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#C#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:15,750 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/b5dc57d5157f4670ad939450a88d5315 is 50, key is test_row_0/C:col10/1733811314759/Put/seqid=0 2024-12-10T06:15:15,755 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/B of 062b93d61f64862ee02e4bde4cb11903 into bb0e250b4af045a0b0c57a5f0fc19c24(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:15,755 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:15,755 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/B, priority=13, startTime=1733811315662; duration=0sec 2024-12-10T06:15:15,755 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:15,755 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:B 2024-12-10T06:15:15,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742097_1273 (size=12493) 2024-12-10T06:15:15,769 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/b5dc57d5157f4670ad939450a88d5315 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/b5dc57d5157f4670ad939450a88d5315 2024-12-10T06:15:15,776 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/C of 062b93d61f64862ee02e4bde4cb11903 into b5dc57d5157f4670ad939450a88d5315(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:15,776 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:15,776 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/C, priority=13, startTime=1733811315675; duration=0sec 2024-12-10T06:15:15,776 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:15,776 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:C 2024-12-10T06:15:15,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T06:15:15,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:15,803 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T06:15:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:15,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/a88bace5501d4937bb86828839116fcf is 50, key is test_row_0/A:col10/1733811314776/Put/seqid=0 2024-12-10T06:15:15,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742098_1274 (size=12151) 2024-12-10T06:15:15,839 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/a88bace5501d4937bb86828839116fcf 2024-12-10T06:15:15,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/56db2896b9fe4c4ab52fa026dea4ae77 is 50, key is test_row_0/B:col10/1733811314776/Put/seqid=0 2024-12-10T06:15:15,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742099_1275 (size=12151) 2024-12-10T06:15:15,866 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/56db2896b9fe4c4ab52fa026dea4ae77 2024-12-10T06:15:15,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/479a9a0ad7694321bb8782a9996662b5 is 50, key is test_row_0/C:col10/1733811314776/Put/seqid=0 2024-12-10T06:15:15,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742100_1276 (size=12151) 2024-12-10T06:15:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:15,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:15,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811375945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811375946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811375946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811375953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811375953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T06:15:16,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811376055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811376065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811376065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811376065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811376065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811376265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811376276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811376276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811376276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811376276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,317 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/479a9a0ad7694321bb8782a9996662b5 2024-12-10T06:15:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/a88bace5501d4937bb86828839116fcf as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a88bace5501d4937bb86828839116fcf 2024-12-10T06:15:16,328 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a88bace5501d4937bb86828839116fcf, entries=150, sequenceid=208, filesize=11.9 K 2024-12-10T06:15:16,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/56db2896b9fe4c4ab52fa026dea4ae77 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/56db2896b9fe4c4ab52fa026dea4ae77 2024-12-10T06:15:16,333 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/56db2896b9fe4c4ab52fa026dea4ae77, entries=150, sequenceid=208, filesize=11.9 K 2024-12-10T06:15:16,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/479a9a0ad7694321bb8782a9996662b5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/479a9a0ad7694321bb8782a9996662b5 2024-12-10T06:15:16,340 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/479a9a0ad7694321bb8782a9996662b5, entries=150, sequenceid=208, filesize=11.9 K 2024-12-10T06:15:16,341 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 062b93d61f64862ee02e4bde4cb11903 in 539ms, sequenceid=208, compaction requested=false 2024-12-10T06:15:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-10T06:15:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-10T06:15:16,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-10T06:15:16,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4620 sec 2024-12-10T06:15:16,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4690 sec 2024-12-10T06:15:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:16,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T06:15:16,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:16,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:16,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:16,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:16,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:16,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:16,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/a0f3eeb11f634bc486131069aaa665a9 is 50, key is test_row_0/A:col10/1733811315943/Put/seqid=0 2024-12-10T06:15:16,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742101_1277 (size=12151) 2024-12-10T06:15:16,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/a0f3eeb11f634bc486131069aaa665a9 2024-12-10T06:15:16,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/b61219cd561d486892deebf133eb468b is 50, key is test_row_0/B:col10/1733811315943/Put/seqid=0 2024-12-10T06:15:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811376599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811376601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811376602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811376604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811376607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742102_1278 (size=12151) 2024-12-10T06:15:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811376715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811376716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811376716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811376716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811376716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811376925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811376925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811376926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811376926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:16,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811376926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T06:15:17,025 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:17,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:17,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/b61219cd561d486892deebf133eb468b 2024-12-10T06:15:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-10T06:15:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T06:15:17,028 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:17,029 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:17,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:17,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/32f7439ae4fd452398f35211dba39279 is 50, key is test_row_0/C:col10/1733811315943/Put/seqid=0 2024-12-10T06:15:17,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742103_1279 (size=12151) 2024-12-10T06:15:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T06:15:17,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T06:15:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:17,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:17,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811377236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811377236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811377236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811377236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811377236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,280 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T06:15:17,280 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T06:15:17,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T06:15:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T06:15:17,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/32f7439ae4fd452398f35211dba39279 2024-12-10T06:15:17,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T06:15:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:17,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/a0f3eeb11f634bc486131069aaa665a9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a0f3eeb11f634bc486131069aaa665a9 2024-12-10T06:15:17,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a0f3eeb11f634bc486131069aaa665a9, entries=150, sequenceid=226, filesize=11.9 K 2024-12-10T06:15:17,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/b61219cd561d486892deebf133eb468b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/b61219cd561d486892deebf133eb468b 2024-12-10T06:15:17,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/b61219cd561d486892deebf133eb468b, entries=150, sequenceid=226, filesize=11.9 K 2024-12-10T06:15:17,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/32f7439ae4fd452398f35211dba39279 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/32f7439ae4fd452398f35211dba39279 2024-12-10T06:15:17,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/32f7439ae4fd452398f35211dba39279, entries=150, sequenceid=226, filesize=11.9 K 2024-12-10T06:15:17,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 062b93d61f64862ee02e4bde4cb11903 in 936ms, sequenceid=226, compaction requested=true 2024-12-10T06:15:17,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:17,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:17,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:17,515 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:17,515 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:17,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:17,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:17,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:17,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:17,517 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:17,517 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:17,517 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/A is initiating minor compaction (all files) 2024-12-10T06:15:17,517 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/B is initiating minor compaction (all files) 2024-12-10T06:15:17,517 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/B in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,517 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/A in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,517 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c91951409c5846348a394789495934e0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a88bace5501d4937bb86828839116fcf, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a0f3eeb11f634bc486131069aaa665a9] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=35.9 K 2024-12-10T06:15:17,517 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/bb0e250b4af045a0b0c57a5f0fc19c24, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/56db2896b9fe4c4ab52fa026dea4ae77, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/b61219cd561d486892deebf133eb468b] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=35.9 K 2024-12-10T06:15:17,520 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c91951409c5846348a394789495934e0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811313616 2024-12-10T06:15:17,520 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting bb0e250b4af045a0b0c57a5f0fc19c24, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811313616 2024-12-10T06:15:17,521 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 56db2896b9fe4c4ab52fa026dea4ae77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733811314776 2024-12-10T06:15:17,521 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting a88bace5501d4937bb86828839116fcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733811314776 2024-12-10T06:15:17,521 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b61219cd561d486892deebf133eb468b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733811315943 2024-12-10T06:15:17,522 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0f3eeb11f634bc486131069aaa665a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733811315943 2024-12-10T06:15:17,538 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#B#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:17,539 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/20a8b18b71654065b963abfbed9954c6 is 50, key is test_row_0/B:col10/1733811315943/Put/seqid=0 2024-12-10T06:15:17,539 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:17,540 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/eae0f46bc6974b7ba7e1acddb5974ee5 is 50, key is test_row_0/A:col10/1733811315943/Put/seqid=0 2024-12-10T06:15:17,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742104_1280 (size=12595) 2024-12-10T06:15:17,559 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/20a8b18b71654065b963abfbed9954c6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/20a8b18b71654065b963abfbed9954c6 2024-12-10T06:15:17,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742105_1281 (size=12595) 2024-12-10T06:15:17,568 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/eae0f46bc6974b7ba7e1acddb5974ee5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/eae0f46bc6974b7ba7e1acddb5974ee5 2024-12-10T06:15:17,573 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/B of 062b93d61f64862ee02e4bde4cb11903 into 20a8b18b71654065b963abfbed9954c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:17,573 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:17,573 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/A of 062b93d61f64862ee02e4bde4cb11903 into eae0f46bc6974b7ba7e1acddb5974ee5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:17,573 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:17,574 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/A, priority=13, startTime=1733811317515; duration=0sec 2024-12-10T06:15:17,574 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:17,574 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:A 2024-12-10T06:15:17,574 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/B, priority=13, startTime=1733811317515; duration=0sec 2024-12-10T06:15:17,574 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:17,574 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:17,574 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:B 2024-12-10T06:15:17,575 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:17,576 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/C is initiating minor compaction (all files) 2024-12-10T06:15:17,576 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/C in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,576 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/b5dc57d5157f4670ad939450a88d5315, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/479a9a0ad7694321bb8782a9996662b5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/32f7439ae4fd452398f35211dba39279] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=35.9 K 2024-12-10T06:15:17,576 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5dc57d5157f4670ad939450a88d5315, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733811313616 2024-12-10T06:15:17,577 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 479a9a0ad7694321bb8782a9996662b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733811314776 2024-12-10T06:15:17,577 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f7439ae4fd452398f35211dba39279, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733811315943 2024-12-10T06:15:17,585 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#C#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:17,585 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/dcf21ac5b25e4c8a832ff67fce06da44 is 50, key is test_row_0/C:col10/1733811315943/Put/seqid=0 2024-12-10T06:15:17,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742106_1282 (size=12595) 2024-12-10T06:15:17,596 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/dcf21ac5b25e4c8a832ff67fce06da44 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/dcf21ac5b25e4c8a832ff67fce06da44 2024-12-10T06:15:17,605 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/C of 062b93d61f64862ee02e4bde4cb11903 into dcf21ac5b25e4c8a832ff67fce06da44(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:17,605 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:17,605 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/C, priority=13, startTime=1733811317516; duration=0sec 2024-12-10T06:15:17,605 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:17,605 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:C 2024-12-10T06:15:17,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T06:15:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:17,646 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T06:15:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:17,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/5057a3b3c40b4e03ad97fe1706ce426d is 50, key is test_row_0/A:col10/1733811316602/Put/seqid=0 2024-12-10T06:15:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T06:15:17,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742107_1283 (size=12151) 2024-12-10T06:15:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:17,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:17,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811377764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811377766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811377768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811377770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811377771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811377875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811377875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811377885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811377886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811377886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/5057a3b3c40b4e03ad97fe1706ce426d 2024-12-10T06:15:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/7b0e6af298f7487c8403bd714a3711b3 is 50, key is test_row_0/B:col10/1733811316602/Put/seqid=0 2024-12-10T06:15:18,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811378085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811378085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811378096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811378096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811378096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742108_1284 (size=12151) 2024-12-10T06:15:18,133 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/7b0e6af298f7487c8403bd714a3711b3 2024-12-10T06:15:18,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/ae0f747bad6a4126b2477940cc625156 is 50, key is test_row_0/C:col10/1733811316602/Put/seqid=0 2024-12-10T06:15:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T06:15:18,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742109_1285 (size=12151) 2024-12-10T06:15:18,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/ae0f747bad6a4126b2477940cc625156 2024-12-10T06:15:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/5057a3b3c40b4e03ad97fe1706ce426d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/5057a3b3c40b4e03ad97fe1706ce426d 2024-12-10T06:15:18,190 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/5057a3b3c40b4e03ad97fe1706ce426d, entries=150, sequenceid=244, filesize=11.9 K 2024-12-10T06:15:18,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/7b0e6af298f7487c8403bd714a3711b3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/7b0e6af298f7487c8403bd714a3711b3 2024-12-10T06:15:18,197 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/7b0e6af298f7487c8403bd714a3711b3, entries=150, sequenceid=244, filesize=11.9 K 2024-12-10T06:15:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/ae0f747bad6a4126b2477940cc625156 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ae0f747bad6a4126b2477940cc625156 2024-12-10T06:15:18,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ae0f747bad6a4126b2477940cc625156, entries=150, sequenceid=244, filesize=11.9 K 2024-12-10T06:15:18,205 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 062b93d61f64862ee02e4bde4cb11903 in 559ms, sequenceid=244, compaction requested=false 2024-12-10T06:15:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-10T06:15:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-10T06:15:18,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-10T06:15:18,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1780 sec 2024-12-10T06:15:18,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1840 sec 2024-12-10T06:15:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:18,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T06:15:18,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:18,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:18,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/01cfce04811d40e49d8c7ba15c7e5bcf is 50, key is test_row_0/A:col10/1733811317769/Put/seqid=0 2024-12-10T06:15:18,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742110_1286 (size=14741) 2024-12-10T06:15:18,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/01cfce04811d40e49d8c7ba15c7e5bcf 2024-12-10T06:15:18,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811378413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811378414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811378416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811378424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/90f7d20fab09400a985c4894d2b21564 is 50, key is test_row_0/B:col10/1733811317769/Put/seqid=0 2024-12-10T06:15:18,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811378426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742111_1287 (size=12301) 2024-12-10T06:15:18,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811378526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811378526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811378535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811378535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811378535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811378735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811378735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811378745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811378745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811378745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:18,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/90f7d20fab09400a985c4894d2b21564 2024-12-10T06:15:18,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/64f0f72f70c9413e9cf15b94b296cf36 is 50, key is test_row_0/C:col10/1733811317769/Put/seqid=0 2024-12-10T06:15:18,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742112_1288 (size=12301) 2024-12-10T06:15:18,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/64f0f72f70c9413e9cf15b94b296cf36 2024-12-10T06:15:18,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/01cfce04811d40e49d8c7ba15c7e5bcf as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/01cfce04811d40e49d8c7ba15c7e5bcf 2024-12-10T06:15:18,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/01cfce04811d40e49d8c7ba15c7e5bcf, entries=200, sequenceid=264, filesize=14.4 K 2024-12-10T06:15:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/90f7d20fab09400a985c4894d2b21564 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/90f7d20fab09400a985c4894d2b21564 2024-12-10T06:15:18,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/90f7d20fab09400a985c4894d2b21564, entries=150, sequenceid=264, filesize=12.0 K 2024-12-10T06:15:18,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/64f0f72f70c9413e9cf15b94b296cf36 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/64f0f72f70c9413e9cf15b94b296cf36 2024-12-10T06:15:18,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/64f0f72f70c9413e9cf15b94b296cf36, entries=150, sequenceid=264, filesize=12.0 K 2024-12-10T06:15:18,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 062b93d61f64862ee02e4bde4cb11903 in 501ms, sequenceid=264, compaction requested=true 2024-12-10T06:15:18,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:18,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:18,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:18,901 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:18,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:18,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:18,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:18,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:18,901 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:18,901 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39487 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:18,902 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/A is initiating minor compaction (all files) 2024-12-10T06:15:18,902 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/A in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:18,902 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/eae0f46bc6974b7ba7e1acddb5974ee5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/5057a3b3c40b4e03ad97fe1706ce426d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/01cfce04811d40e49d8c7ba15c7e5bcf] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=38.6 K 2024-12-10T06:15:18,902 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:18,902 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/B is initiating minor compaction (all files) 2024-12-10T06:15:18,902 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/B in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:18,902 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae0f46bc6974b7ba7e1acddb5974ee5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733811315943 2024-12-10T06:15:18,902 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/20a8b18b71654065b963abfbed9954c6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/7b0e6af298f7487c8403bd714a3711b3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/90f7d20fab09400a985c4894d2b21564] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=36.2 K 2024-12-10T06:15:18,902 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a8b18b71654065b963abfbed9954c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733811315943 2024-12-10T06:15:18,903 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5057a3b3c40b4e03ad97fe1706ce426d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733811316598 2024-12-10T06:15:18,904 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0e6af298f7487c8403bd714a3711b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733811316598 2024-12-10T06:15:18,904 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01cfce04811d40e49d8c7ba15c7e5bcf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733811317768 2024-12-10T06:15:18,904 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f7d20fab09400a985c4894d2b21564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733811317769 2024-12-10T06:15:18,915 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:18,915 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#A#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:18,915 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/15734191d62c43d189815562ba8b45e2 is 50, key is test_row_0/B:col10/1733811317769/Put/seqid=0 2024-12-10T06:15:18,915 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/315a67c2dc1543ebacd492a6cc754bf3 is 50, key is test_row_0/A:col10/1733811317769/Put/seqid=0 2024-12-10T06:15:18,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742113_1289 (size=12847) 2024-12-10T06:15:18,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742114_1290 (size=12847) 2024-12-10T06:15:18,935 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/315a67c2dc1543ebacd492a6cc754bf3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/315a67c2dc1543ebacd492a6cc754bf3 2024-12-10T06:15:18,940 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/A of 062b93d61f64862ee02e4bde4cb11903 into 315a67c2dc1543ebacd492a6cc754bf3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:18,940 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:18,940 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/A, priority=13, startTime=1733811318900; duration=0sec 2024-12-10T06:15:18,940 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:18,940 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:A 2024-12-10T06:15:18,940 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:18,941 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:18,941 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/C is initiating minor compaction (all files) 2024-12-10T06:15:18,941 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/C in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:18,942 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/dcf21ac5b25e4c8a832ff67fce06da44, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ae0f747bad6a4126b2477940cc625156, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/64f0f72f70c9413e9cf15b94b296cf36] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=36.2 K 2024-12-10T06:15:18,942 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf21ac5b25e4c8a832ff67fce06da44, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733811315943 2024-12-10T06:15:18,942 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0f747bad6a4126b2477940cc625156, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733811316598 2024-12-10T06:15:18,943 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64f0f72f70c9413e9cf15b94b296cf36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733811317769 2024-12-10T06:15:18,949 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:18,949 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/8e99ec692d42430c8df0d41e4dc7c854 is 50, key is test_row_0/C:col10/1733811317769/Put/seqid=0 2024-12-10T06:15:18,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742115_1291 (size=12847) 2024-12-10T06:15:18,966 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/8e99ec692d42430c8df0d41e4dc7c854 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/8e99ec692d42430c8df0d41e4dc7c854 2024-12-10T06:15:18,971 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/C of 062b93d61f64862ee02e4bde4cb11903 into 8e99ec692d42430c8df0d41e4dc7c854(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:18,971 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:18,971 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/C, priority=13, startTime=1733811318901; duration=0sec 2024-12-10T06:15:18,971 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:18,971 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:C 2024-12-10T06:15:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:19,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T06:15:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:19,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/9e27761b4e5f4b128727066e6c18d90d is 50, key is test_row_0/A:col10/1733811319047/Put/seqid=0 2024-12-10T06:15:19,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742116_1292 (size=14741) 2024-12-10T06:15:19,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/9e27761b4e5f4b128727066e6c18d90d 2024-12-10T06:15:19,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/9af021563bec4f04b4f17ef2b7ed4df6 is 50, key is test_row_0/B:col10/1733811319047/Put/seqid=0 2024-12-10T06:15:19,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811379072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742117_1293 (size=12301) 2024-12-10T06:15:19,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811379077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811379077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811379078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811379078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T06:15:19,176 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:19,176 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-10T06:15:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T06:15:19,178 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:19,179 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:19,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:19,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811379185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811379195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811379195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811379195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811379195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T06:15:19,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T06:15:19,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:19,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,337 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/15734191d62c43d189815562ba8b45e2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/15734191d62c43d189815562ba8b45e2 2024-12-10T06:15:19,342 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/B of 062b93d61f64862ee02e4bde4cb11903 into 15734191d62c43d189815562ba8b45e2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:19,342 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:19,342 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/B, priority=13, startTime=1733811318901; duration=0sec 2024-12-10T06:15:19,342 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:19,342 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:B 2024-12-10T06:15:19,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811379395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811379405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811379405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811379405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811379405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/9af021563bec4f04b4f17ef2b7ed4df6 2024-12-10T06:15:19,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T06:15:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:19,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/1d1a7753e6a94b8098742fc7cc443889 is 50, key is test_row_0/C:col10/1733811319047/Put/seqid=0 2024-12-10T06:15:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T06:15:19,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742118_1294 (size=12301) 2024-12-10T06:15:19,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T06:15:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:19,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811379706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811379715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811379716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811379715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811379716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:19,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T06:15:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:19,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T06:15:19,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/1d1a7753e6a94b8098742fc7cc443889 2024-12-10T06:15:19,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/9e27761b4e5f4b128727066e6c18d90d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/9e27761b4e5f4b128727066e6c18d90d 2024-12-10T06:15:19,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/9e27761b4e5f4b128727066e6c18d90d, entries=200, sequenceid=281, filesize=14.4 K 2024-12-10T06:15:19,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/9af021563bec4f04b4f17ef2b7ed4df6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/9af021563bec4f04b4f17ef2b7ed4df6 2024-12-10T06:15:19,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/9af021563bec4f04b4f17ef2b7ed4df6, entries=150, sequenceid=281, filesize=12.0 K 2024-12-10T06:15:19,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/1d1a7753e6a94b8098742fc7cc443889 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/1d1a7753e6a94b8098742fc7cc443889 2024-12-10T06:15:19,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/1d1a7753e6a94b8098742fc7cc443889, entries=150, sequenceid=281, filesize=12.0 K 2024-12-10T06:15:19,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 062b93d61f64862ee02e4bde4cb11903 in 889ms, sequenceid=281, compaction requested=false 2024-12-10T06:15:19,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:19,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T06:15:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:19,949 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T06:15:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:19,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/f9db83632e8840d285a4e16ad4a20e1d is 50, key is test_row_0/A:col10/1733811319076/Put/seqid=0 2024-12-10T06:15:19,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742119_1295 (size=12301) 2024-12-10T06:15:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:20,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:20,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811380236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811380236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811380237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811380239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811380241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T06:15:20,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811380355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811380355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811380355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811380355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,360 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/f9db83632e8840d285a4e16ad4a20e1d 2024-12-10T06:15:20,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/a5e2de423fc94fed9bf7df201f799de7 is 50, key is test_row_0/B:col10/1733811319076/Put/seqid=0 2024-12-10T06:15:20,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742120_1296 (size=12301) 2024-12-10T06:15:20,371 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/a5e2de423fc94fed9bf7df201f799de7 2024-12-10T06:15:20,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/30e37925d6e540cda62ce3a0c4250074 is 50, key is test_row_0/C:col10/1733811319076/Put/seqid=0 2024-12-10T06:15:20,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742121_1297 (size=12301) 2024-12-10T06:15:20,398 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/30e37925d6e540cda62ce3a0c4250074 2024-12-10T06:15:20,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/f9db83632e8840d285a4e16ad4a20e1d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f9db83632e8840d285a4e16ad4a20e1d 2024-12-10T06:15:20,413 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f9db83632e8840d285a4e16ad4a20e1d, entries=150, sequenceid=300, filesize=12.0 K 2024-12-10T06:15:20,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/a5e2de423fc94fed9bf7df201f799de7 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a5e2de423fc94fed9bf7df201f799de7 2024-12-10T06:15:20,420 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a5e2de423fc94fed9bf7df201f799de7, entries=150, sequenceid=300, filesize=12.0 K 2024-12-10T06:15:20,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/30e37925d6e540cda62ce3a0c4250074 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/30e37925d6e540cda62ce3a0c4250074 2024-12-10T06:15:20,426 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/30e37925d6e540cda62ce3a0c4250074, entries=150, sequenceid=300, filesize=12.0 K 2024-12-10T06:15:20,427 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 062b93d61f64862ee02e4bde4cb11903 in 478ms, sequenceid=300, compaction requested=true 2024-12-10T06:15:20,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:20,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:20,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-10T06:15:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-10T06:15:20,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-10T06:15:20,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2500 sec 2024-12-10T06:15:20,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2570 sec 2024-12-10T06:15:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:20,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T06:15:20,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:20,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:20,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:20,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:20,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:20,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:20,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/63c6d9cc9e584aff915557e3d3c2c8dc is 50, key is test_row_0/A:col10/1733811320238/Put/seqid=0 2024-12-10T06:15:20,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742122_1298 (size=12301) 2024-12-10T06:15:20,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811380593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811380594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811380595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811380595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811380705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811380706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811380706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811380706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811380915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811380915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811380915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811380935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:20,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/63c6d9cc9e584aff915557e3d3c2c8dc 2024-12-10T06:15:20,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/3934de8299d14fdb85df51503b6e8de2 is 50, key is test_row_0/B:col10/1733811320238/Put/seqid=0 2024-12-10T06:15:20,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742123_1299 (size=12301) 2024-12-10T06:15:21,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811381225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811381225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811381225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:21,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811381245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:21,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811381256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:21,261 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 062b93d61f64862ee02e4bde4cb11903 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T06:15:21,326 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:21,326 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-10T06:15:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T06:15:21,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:21,329 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:21,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:21,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/3934de8299d14fdb85df51503b6e8de2 2024-12-10T06:15:21,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/a65ec64e79ef400cb26e264dc9aaaf77 is 50, key is test_row_0/C:col10/1733811320238/Put/seqid=0 2024-12-10T06:15:21,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742124_1300 (size=12301) 2024-12-10T06:15:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T06:15:21,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T06:15:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:21,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:21,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T06:15:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T06:15:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811381736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:21,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811381736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811381745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:21,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811381755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:21,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T06:15:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:21,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/a65ec64e79ef400cb26e264dc9aaaf77 2024-12-10T06:15:21,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/63c6d9cc9e584aff915557e3d3c2c8dc as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/63c6d9cc9e584aff915557e3d3c2c8dc 2024-12-10T06:15:21,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/63c6d9cc9e584aff915557e3d3c2c8dc, entries=150, sequenceid=315, filesize=12.0 K 2024-12-10T06:15:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/3934de8299d14fdb85df51503b6e8de2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/3934de8299d14fdb85df51503b6e8de2 2024-12-10T06:15:21,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/3934de8299d14fdb85df51503b6e8de2, entries=150, sequenceid=315, filesize=12.0 K 2024-12-10T06:15:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/a65ec64e79ef400cb26e264dc9aaaf77 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a65ec64e79ef400cb26e264dc9aaaf77 2024-12-10T06:15:21,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a65ec64e79ef400cb26e264dc9aaaf77, entries=150, sequenceid=315, filesize=12.0 K 2024-12-10T06:15:21,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 062b93d61f64862ee02e4bde4cb11903 in 1255ms, sequenceid=315, compaction requested=true 2024-12-10T06:15:21,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:21,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:21,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:21,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:21,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:21,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:21,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:21,824 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:21,824 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:21,825 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:21,825 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:21,825 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/A is initiating minor compaction (all files) 2024-12-10T06:15:21,825 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/B is initiating minor compaction (all files) 2024-12-10T06:15:21,825 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/A in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,825 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/B in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,825 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/15734191d62c43d189815562ba8b45e2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/9af021563bec4f04b4f17ef2b7ed4df6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a5e2de423fc94fed9bf7df201f799de7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/3934de8299d14fdb85df51503b6e8de2] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=48.6 K 2024-12-10T06:15:21,825 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/315a67c2dc1543ebacd492a6cc754bf3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/9e27761b4e5f4b128727066e6c18d90d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f9db83632e8840d285a4e16ad4a20e1d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/63c6d9cc9e584aff915557e3d3c2c8dc] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=51.0 K 2024-12-10T06:15:21,825 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 15734191d62c43d189815562ba8b45e2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733811317769 2024-12-10T06:15:21,825 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 315a67c2dc1543ebacd492a6cc754bf3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733811317769 2024-12-10T06:15:21,826 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e27761b4e5f4b128727066e6c18d90d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733811318410 2024-12-10T06:15:21,826 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af021563bec4f04b4f17ef2b7ed4df6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733811318410 2024-12-10T06:15:21,826 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9db83632e8840d285a4e16ad4a20e1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733811319068 2024-12-10T06:15:21,826 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e2de423fc94fed9bf7df201f799de7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733811319068 2024-12-10T06:15:21,826 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 3934de8299d14fdb85df51503b6e8de2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733811320230 2024-12-10T06:15:21,826 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63c6d9cc9e584aff915557e3d3c2c8dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733811320230 2024-12-10T06:15:21,841 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:21,842 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/6f02327ad57d4ac496407631706e04cc is 50, key is test_row_0/B:col10/1733811320238/Put/seqid=0 2024-12-10T06:15:21,843 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#A#compaction#253 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:21,844 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/cd25f0ccf920408981c2c6b8f49ba427 is 50, key is test_row_0/A:col10/1733811320238/Put/seqid=0 2024-12-10T06:15:21,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742126_1302 (size=12983) 2024-12-10T06:15:21,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742125_1301 (size=12983) 2024-12-10T06:15:21,855 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/6f02327ad57d4ac496407631706e04cc as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6f02327ad57d4ac496407631706e04cc 2024-12-10T06:15:21,855 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/cd25f0ccf920408981c2c6b8f49ba427 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/cd25f0ccf920408981c2c6b8f49ba427 2024-12-10T06:15:21,860 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/B of 062b93d61f64862ee02e4bde4cb11903 into 6f02327ad57d4ac496407631706e04cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:21,860 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:21,860 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/B, priority=12, startTime=1733811321824; duration=0sec 2024-12-10T06:15:21,860 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/A of 062b93d61f64862ee02e4bde4cb11903 into cd25f0ccf920408981c2c6b8f49ba427(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:21,860 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:21,860 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/A, priority=12, startTime=1733811321824; duration=0sec 2024-12-10T06:15:21,860 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:21,860 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:B 2024-12-10T06:15:21,860 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:21,860 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:A 2024-12-10T06:15:21,860 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:21,862 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:21,862 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/C is initiating minor compaction (all files) 2024-12-10T06:15:21,862 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/C in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,862 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/8e99ec692d42430c8df0d41e4dc7c854, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/1d1a7753e6a94b8098742fc7cc443889, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/30e37925d6e540cda62ce3a0c4250074, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a65ec64e79ef400cb26e264dc9aaaf77] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=48.6 K 2024-12-10T06:15:21,863 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e99ec692d42430c8df0d41e4dc7c854, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733811317769 2024-12-10T06:15:21,863 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d1a7753e6a94b8098742fc7cc443889, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733811318410 2024-12-10T06:15:21,863 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 30e37925d6e540cda62ce3a0c4250074, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733811319068 2024-12-10T06:15:21,863 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting a65ec64e79ef400cb26e264dc9aaaf77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733811320230 2024-12-10T06:15:21,872 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#C#compaction#254 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:21,872 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c6e942db0f57456e87bf84145365c754 is 50, key is test_row_0/C:col10/1733811320238/Put/seqid=0 2024-12-10T06:15:21,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742127_1303 (size=12983) 2024-12-10T06:15:21,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T06:15:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:21,941 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T06:15:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:21,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/bdb1c069731547c0a1bfaa8e7483b24e is 50, key is test_row_0/A:col10/1733811320593/Put/seqid=0 2024-12-10T06:15:21,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742128_1304 (size=12301) 2024-12-10T06:15:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T06:15:22,293 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c6e942db0f57456e87bf84145365c754 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c6e942db0f57456e87bf84145365c754 2024-12-10T06:15:22,298 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/C of 062b93d61f64862ee02e4bde4cb11903 into c6e942db0f57456e87bf84145365c754(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:22,299 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:22,299 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/C, priority=12, startTime=1733811321824; duration=0sec 2024-12-10T06:15:22,299 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:22,299 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:C 2024-12-10T06:15:22,352 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/bdb1c069731547c0a1bfaa8e7483b24e 2024-12-10T06:15:22,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/753ff9e8865a43feab978137e3e67b58 is 50, key is test_row_0/B:col10/1733811320593/Put/seqid=0 2024-12-10T06:15:22,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742129_1305 (size=12301) 2024-12-10T06:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T06:15:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:22,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:22,764 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/753ff9e8865a43feab978137e3e67b58 2024-12-10T06:15:22,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/720b0c3792084cb58ddd7ca3eb04b1e9 is 50, key is test_row_0/C:col10/1733811320593/Put/seqid=0 2024-12-10T06:15:22,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742130_1306 (size=12301) 2024-12-10T06:15:22,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811382799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:22,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811382799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:22,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811382801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:22,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811382805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811382916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811382916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811382916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811382916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811383126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811383126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811383126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811383126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,176 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/720b0c3792084cb58ddd7ca3eb04b1e9 2024-12-10T06:15:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/bdb1c069731547c0a1bfaa8e7483b24e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bdb1c069731547c0a1bfaa8e7483b24e 2024-12-10T06:15:23,185 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bdb1c069731547c0a1bfaa8e7483b24e, entries=150, sequenceid=335, filesize=12.0 K 2024-12-10T06:15:23,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/753ff9e8865a43feab978137e3e67b58 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/753ff9e8865a43feab978137e3e67b58 2024-12-10T06:15:23,190 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/753ff9e8865a43feab978137e3e67b58, entries=150, sequenceid=335, filesize=12.0 K 2024-12-10T06:15:23,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/720b0c3792084cb58ddd7ca3eb04b1e9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/720b0c3792084cb58ddd7ca3eb04b1e9 2024-12-10T06:15:23,195 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/720b0c3792084cb58ddd7ca3eb04b1e9, entries=150, sequenceid=335, filesize=12.0 K 2024-12-10T06:15:23,195 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 062b93d61f64862ee02e4bde4cb11903 in 1254ms, sequenceid=335, compaction requested=false 2024-12-10T06:15:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-10T06:15:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-10T06:15:23,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-10T06:15:23,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8680 sec 2024-12-10T06:15:23,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.8730 sec 2024-12-10T06:15:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:23,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T06:15:23,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:23,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7c37d943373149f2972fbd560688a230 is 50, key is test_row_0/A:col10/1733811322797/Put/seqid=0 2024-12-10T06:15:23,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742131_1307 (size=14741) 2024-12-10T06:15:23,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811383355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811383436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811383436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811383436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811383436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811383465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T06:15:23,476 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:23,476 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-10T06:15:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T06:15:23,478 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:23,479 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:23,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T06:15:23,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T06:15:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:23,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:23,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811383675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7c37d943373149f2972fbd560688a230 2024-12-10T06:15:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/17be7fa1abf947bab51f9e588e4243a1 is 50, key is test_row_0/B:col10/1733811322797/Put/seqid=0 2024-12-10T06:15:23,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742132_1308 (size=12301) 2024-12-10T06:15:23,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T06:15:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:23,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T06:15:23,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T06:15:23,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:23,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:23,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:23,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811383966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811383966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811383966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811383966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811383986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:24,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T06:15:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:24,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:24,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/17be7fa1abf947bab51f9e588e4243a1 2024-12-10T06:15:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T06:15:24,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c990af843f8b4f5cbe4bbd6bab7f129b is 50, key is test_row_0/C:col10/1733811322797/Put/seqid=0 2024-12-10T06:15:24,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742133_1309 (size=12301) 2024-12-10T06:15:24,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T06:15:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:24,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T06:15:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:24,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811384505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:24,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c990af843f8b4f5cbe4bbd6bab7f129b 2024-12-10T06:15:24,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7c37d943373149f2972fbd560688a230 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7c37d943373149f2972fbd560688a230 2024-12-10T06:15:24,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7c37d943373149f2972fbd560688a230, entries=200, sequenceid=351, filesize=14.4 K 2024-12-10T06:15:24,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/17be7fa1abf947bab51f9e588e4243a1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/17be7fa1abf947bab51f9e588e4243a1 2024-12-10T06:15:24,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/17be7fa1abf947bab51f9e588e4243a1, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T06:15:24,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c990af843f8b4f5cbe4bbd6bab7f129b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c990af843f8b4f5cbe4bbd6bab7f129b 2024-12-10T06:15:24,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c990af843f8b4f5cbe4bbd6bab7f129b, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T06:15:24,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 062b93d61f64862ee02e4bde4cb11903 in 1273ms, sequenceid=351, compaction requested=true 2024-12-10T06:15:24,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:24,542 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:24,544 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:24,544 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/A is initiating minor compaction (all files) 2024-12-10T06:15:24,544 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/A in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,544 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/cd25f0ccf920408981c2c6b8f49ba427, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bdb1c069731547c0a1bfaa8e7483b24e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7c37d943373149f2972fbd560688a230] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=39.1 K 2024-12-10T06:15:24,544 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:24,544 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd25f0ccf920408981c2c6b8f49ba427, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733811320230 2024-12-10T06:15:24,545 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdb1c069731547c0a1bfaa8e7483b24e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733811320592 2024-12-10T06:15:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:24,545 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c37d943373149f2972fbd560688a230, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733811322797 2024-12-10T06:15:24,545 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:24,545 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/B is initiating minor compaction (all files) 2024-12-10T06:15:24,545 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/B in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,546 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6f02327ad57d4ac496407631706e04cc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/753ff9e8865a43feab978137e3e67b58, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/17be7fa1abf947bab51f9e588e4243a1] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=36.7 K 2024-12-10T06:15:24,546 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f02327ad57d4ac496407631706e04cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733811320230 2024-12-10T06:15:24,546 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 753ff9e8865a43feab978137e3e67b58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733811320592 2024-12-10T06:15:24,547 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 17be7fa1abf947bab51f9e588e4243a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733811322797 2024-12-10T06:15:24,555 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:24,556 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/c019e6caecef4476a6b4692ee758dd56 is 50, key is test_row_0/A:col10/1733811322797/Put/seqid=0 2024-12-10T06:15:24,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T06:15:24,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,557 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T06:15:24,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:24,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:24,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:24,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:24,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:24,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:24,563 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:24,564 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/1198b030d50f4c3ebf48dc13f783ebfc is 50, key is test_row_0/B:col10/1733811322797/Put/seqid=0 2024-12-10T06:15:24,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742134_1310 (size=13085) 2024-12-10T06:15:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/3edccb92399c4ab488e9c9446a6f0c3b is 50, key is test_row_0/A:col10/1733811323319/Put/seqid=0 2024-12-10T06:15:24,582 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/c019e6caecef4476a6b4692ee758dd56 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c019e6caecef4476a6b4692ee758dd56 2024-12-10T06:15:24,591 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/A of 062b93d61f64862ee02e4bde4cb11903 into c019e6caecef4476a6b4692ee758dd56(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:24,591 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:24,591 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/A, priority=13, startTime=1733811324542; duration=0sec 2024-12-10T06:15:24,591 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:24,591 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:A 2024-12-10T06:15:24,591 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:24,594 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:24,595 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/C is initiating minor compaction (all files) 2024-12-10T06:15:24,595 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/C in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:24,595 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c6e942db0f57456e87bf84145365c754, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/720b0c3792084cb58ddd7ca3eb04b1e9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c990af843f8b4f5cbe4bbd6bab7f129b] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=36.7 K 2024-12-10T06:15:24,595 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6e942db0f57456e87bf84145365c754, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733811320230 2024-12-10T06:15:24,596 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 720b0c3792084cb58ddd7ca3eb04b1e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733811320592 2024-12-10T06:15:24,596 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c990af843f8b4f5cbe4bbd6bab7f129b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733811322797 2024-12-10T06:15:24,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742136_1312 (size=12301) 2024-12-10T06:15:24,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742135_1311 (size=13085) 2024-12-10T06:15:24,611 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:24,611 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/2950d0a6f9f54efea079cabe1e058616 is 50, key is test_row_0/C:col10/1733811322797/Put/seqid=0 2024-12-10T06:15:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T06:15:24,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742137_1313 (size=13085) 2024-12-10T06:15:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:24,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:24,998 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/3edccb92399c4ab488e9c9446a6f0c3b 2024-12-10T06:15:25,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811384997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811384997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811384998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811385000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/c91a964177604e4e82896e77a410c176 is 50, key is test_row_0/B:col10/1733811323319/Put/seqid=0 2024-12-10T06:15:25,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742138_1314 (size=12301) 2024-12-10T06:15:25,019 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/1198b030d50f4c3ebf48dc13f783ebfc as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/1198b030d50f4c3ebf48dc13f783ebfc 2024-12-10T06:15:25,025 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/B of 062b93d61f64862ee02e4bde4cb11903 into 1198b030d50f4c3ebf48dc13f783ebfc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:25,025 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:25,025 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/B, priority=13, startTime=1733811324544; duration=0sec 2024-12-10T06:15:25,025 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:25,026 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:B 2024-12-10T06:15:25,030 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/2950d0a6f9f54efea079cabe1e058616 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/2950d0a6f9f54efea079cabe1e058616 2024-12-10T06:15:25,037 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/C of 062b93d61f64862ee02e4bde4cb11903 into 2950d0a6f9f54efea079cabe1e058616(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:25,037 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:25,037 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/C, priority=13, startTime=1733811324545; duration=0sec 2024-12-10T06:15:25,037 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:25,037 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:C 2024-12-10T06:15:25,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811385115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811385115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811385115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811385115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811385325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811385325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811385326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811385326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,412 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/c91a964177604e4e82896e77a410c176 2024-12-10T06:15:25,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/91d0fb88f5c54be4b3a56ebb7f36060b is 50, key is test_row_0/C:col10/1733811323319/Put/seqid=0 2024-12-10T06:15:25,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742139_1315 (size=12301) 2024-12-10T06:15:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811385525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,529 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 062b93d61f64862ee02e4bde4cb11903 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T06:15:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811385635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811385635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811385635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811385636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:25,825 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/91d0fb88f5c54be4b3a56ebb7f36060b 2024-12-10T06:15:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/3edccb92399c4ab488e9c9446a6f0c3b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/3edccb92399c4ab488e9c9446a6f0c3b 2024-12-10T06:15:25,838 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/3edccb92399c4ab488e9c9446a6f0c3b, entries=150, sequenceid=368, filesize=12.0 K 2024-12-10T06:15:25,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/c91a964177604e4e82896e77a410c176 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/c91a964177604e4e82896e77a410c176 2024-12-10T06:15:25,844 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/c91a964177604e4e82896e77a410c176, entries=150, sequenceid=368, filesize=12.0 K 2024-12-10T06:15:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/91d0fb88f5c54be4b3a56ebb7f36060b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/91d0fb88f5c54be4b3a56ebb7f36060b 2024-12-10T06:15:25,851 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/91d0fb88f5c54be4b3a56ebb7f36060b, entries=150, sequenceid=368, filesize=12.0 K 2024-12-10T06:15:25,852 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 062b93d61f64862ee02e4bde4cb11903 in 1295ms, sequenceid=368, compaction requested=false 2024-12-10T06:15:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-10T06:15:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-10T06:15:25,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-10T06:15:25,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3780 sec 2024-12-10T06:15:25,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.3840 sec 2024-12-10T06:15:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:26,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:15:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:26,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/ce983e93e3ff423d94747d43be00fa91 is 50, key is test_row_0/A:col10/1733811324987/Put/seqid=0 2024-12-10T06:15:26,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742140_1316 (size=12301) 2024-12-10T06:15:26,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/ce983e93e3ff423d94747d43be00fa91 2024-12-10T06:15:26,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/86e247bf38a349ce99a5b700368d6cc6 is 50, key is test_row_0/B:col10/1733811324987/Put/seqid=0 2024-12-10T06:15:26,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811386162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811386164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811386165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742141_1317 (size=12301) 2024-12-10T06:15:26,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811386169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811386275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811386276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811386276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811386285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811386486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811386486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811386486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811386495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/86e247bf38a349ce99a5b700368d6cc6 2024-12-10T06:15:26,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/ff1cc3dcaf42445eba7d88f7c142ce09 is 50, key is test_row_0/C:col10/1733811324987/Put/seqid=0 2024-12-10T06:15:26,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742142_1318 (size=12301) 2024-12-10T06:15:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811386795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811386795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811386795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811386805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:26,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/ff1cc3dcaf42445eba7d88f7c142ce09 2024-12-10T06:15:26,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/ce983e93e3ff423d94747d43be00fa91 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ce983e93e3ff423d94747d43be00fa91 2024-12-10T06:15:26,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ce983e93e3ff423d94747d43be00fa91, entries=150, sequenceid=388, filesize=12.0 K 2024-12-10T06:15:26,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/86e247bf38a349ce99a5b700368d6cc6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/86e247bf38a349ce99a5b700368d6cc6 2024-12-10T06:15:27,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/86e247bf38a349ce99a5b700368d6cc6, entries=150, sequenceid=388, filesize=12.0 K 2024-12-10T06:15:27,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/ff1cc3dcaf42445eba7d88f7c142ce09 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ff1cc3dcaf42445eba7d88f7c142ce09 2024-12-10T06:15:27,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ff1cc3dcaf42445eba7d88f7c142ce09, entries=150, sequenceid=388, filesize=12.0 K 2024-12-10T06:15:27,027 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T06:15:27,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 062b93d61f64862ee02e4bde4cb11903 in 881ms, sequenceid=388, compaction requested=true 2024-12-10T06:15:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:27,028 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:27,028 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:27,031 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:27,031 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/A is initiating minor compaction (all files) 2024-12-10T06:15:27,031 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/A in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:27,031 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c019e6caecef4476a6b4692ee758dd56, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/3edccb92399c4ab488e9c9446a6f0c3b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ce983e93e3ff423d94747d43be00fa91] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=36.8 K 2024-12-10T06:15:27,031 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:27,031 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/B is initiating minor compaction (all files) 2024-12-10T06:15:27,031 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/B in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:27,032 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/1198b030d50f4c3ebf48dc13f783ebfc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/c91a964177604e4e82896e77a410c176, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/86e247bf38a349ce99a5b700368d6cc6] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=36.8 K 2024-12-10T06:15:27,032 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c019e6caecef4476a6b4692ee758dd56, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733811322797 2024-12-10T06:15:27,032 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3edccb92399c4ab488e9c9446a6f0c3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733811323319 2024-12-10T06:15:27,032 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 1198b030d50f4c3ebf48dc13f783ebfc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733811322797 2024-12-10T06:15:27,032 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce983e93e3ff423d94747d43be00fa91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733811324987 2024-12-10T06:15:27,033 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting c91a964177604e4e82896e77a410c176, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733811323319 2024-12-10T06:15:27,033 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 86e247bf38a349ce99a5b700368d6cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733811324987 2024-12-10T06:15:27,048 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:27,048 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/2097cdb9be45462399930448034de2c6 is 50, key is test_row_0/B:col10/1733811324987/Put/seqid=0 2024-12-10T06:15:27,050 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:27,051 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/60dec58905ce4ec7b374c5ee118c103c is 50, key is test_row_0/A:col10/1733811324987/Put/seqid=0 2024-12-10T06:15:27,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742144_1320 (size=13187) 2024-12-10T06:15:27,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742143_1319 (size=13187) 2024-12-10T06:15:27,115 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/2097cdb9be45462399930448034de2c6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/2097cdb9be45462399930448034de2c6 2024-12-10T06:15:27,128 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/B of 062b93d61f64862ee02e4bde4cb11903 into 2097cdb9be45462399930448034de2c6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:27,128 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:27,128 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/B, priority=13, startTime=1733811327028; duration=0sec 2024-12-10T06:15:27,129 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:27,129 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:B 2024-12-10T06:15:27,129 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:27,130 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:27,130 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/C is initiating minor compaction (all files) 2024-12-10T06:15:27,130 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/C in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:27,130 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/2950d0a6f9f54efea079cabe1e058616, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/91d0fb88f5c54be4b3a56ebb7f36060b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ff1cc3dcaf42445eba7d88f7c142ce09] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=36.8 K 2024-12-10T06:15:27,131 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2950d0a6f9f54efea079cabe1e058616, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733811322797 2024-12-10T06:15:27,132 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d0fb88f5c54be4b3a56ebb7f36060b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733811323319 2024-12-10T06:15:27,132 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1cc3dcaf42445eba7d88f7c142ce09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733811324987 2024-12-10T06:15:27,147 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:27,147 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/e0abd65a7c914041b460148e5971073f is 50, key is test_row_0/C:col10/1733811324987/Put/seqid=0 2024-12-10T06:15:27,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742145_1321 (size=13187) 2024-12-10T06:15:27,194 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/e0abd65a7c914041b460148e5971073f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/e0abd65a7c914041b460148e5971073f 2024-12-10T06:15:27,210 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/C of 062b93d61f64862ee02e4bde4cb11903 into e0abd65a7c914041b460148e5971073f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:27,210 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:27,210 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/C, priority=13, startTime=1733811327028; duration=0sec 2024-12-10T06:15:27,210 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:27,210 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:C 2024-12-10T06:15:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:27,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:15:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:27,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:27,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/f4d0004bd72841a59d4f3cae6dd2d750 is 50, key is test_row_0/A:col10/1733811326156/Put/seqid=0 2024-12-10T06:15:27,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742146_1322 (size=12301) 2024-12-10T06:15:27,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/f4d0004bd72841a59d4f3cae6dd2d750 2024-12-10T06:15:27,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/108a4d32b9c04b52ad7d6802811a33f1 is 50, key is test_row_0/B:col10/1733811326156/Put/seqid=0 2024-12-10T06:15:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811387340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811387356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811387356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811387357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742147_1323 (size=12301) 2024-12-10T06:15:27,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/108a4d32b9c04b52ad7d6802811a33f1 2024-12-10T06:15:27,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/425490b8c8d14ff68b3a0a4248086494 is 50, key is test_row_0/C:col10/1733811326156/Put/seqid=0 2024-12-10T06:15:27,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742148_1324 (size=12301) 2024-12-10T06:15:27,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811387465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811387475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811387475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811387475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,489 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/60dec58905ce4ec7b374c5ee118c103c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/60dec58905ce4ec7b374c5ee118c103c 2024-12-10T06:15:27,495 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/A of 062b93d61f64862ee02e4bde4cb11903 into 60dec58905ce4ec7b374c5ee118c103c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:27,495 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:27,495 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/A, priority=13, startTime=1733811327028; duration=0sec 2024-12-10T06:15:27,495 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:27,495 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:A 2024-12-10T06:15:27,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46882 deadline: 1733811387556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,561 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 062b93d61f64862ee02e4bde4cb11903 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:27,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T06:15:27,646 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:27,647 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-10T06:15:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T06:15:27,649 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:27,649 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:27,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:27,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811387676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811387686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811387686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811387687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T06:15:27,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T06:15:27,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:27,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:27,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:27,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:27,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:27,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/425490b8c8d14ff68b3a0a4248086494 2024-12-10T06:15:27,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/f4d0004bd72841a59d4f3cae6dd2d750 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f4d0004bd72841a59d4f3cae6dd2d750 2024-12-10T06:15:27,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f4d0004bd72841a59d4f3cae6dd2d750, entries=150, sequenceid=406, filesize=12.0 K 2024-12-10T06:15:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/108a4d32b9c04b52ad7d6802811a33f1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/108a4d32b9c04b52ad7d6802811a33f1 2024-12-10T06:15:27,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/108a4d32b9c04b52ad7d6802811a33f1, entries=150, sequenceid=406, filesize=12.0 K 2024-12-10T06:15:27,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/425490b8c8d14ff68b3a0a4248086494 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/425490b8c8d14ff68b3a0a4248086494 2024-12-10T06:15:27,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/425490b8c8d14ff68b3a0a4248086494, entries=150, sequenceid=406, filesize=12.0 K 2024-12-10T06:15:27,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 062b93d61f64862ee02e4bde4cb11903 in 538ms, sequenceid=406, compaction requested=false 2024-12-10T06:15:27,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:27,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T06:15:27,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:27,959 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:15:27,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:27,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:27,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:27,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:27,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:27,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T06:15:27,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7ee5dc9431bd428888e6c4c548a27cf5 is 50, key is test_row_0/A:col10/1733811327341/Put/seqid=0 2024-12-10T06:15:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:27,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. as already flushing 2024-12-10T06:15:27,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742149_1325 (size=12301) 2024-12-10T06:15:28,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811388012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811388016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811388016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811388019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811388125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811388126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811388135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811388136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T06:15:28,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811388336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811388336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811388345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811388345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,392 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7ee5dc9431bd428888e6c4c548a27cf5 2024-12-10T06:15:28,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/ab8ece747fcf4a588538ac8e7a15ab16 is 50, key is test_row_0/B:col10/1733811327341/Put/seqid=0 2024-12-10T06:15:28,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742150_1326 (size=12301) 2024-12-10T06:15:28,425 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/ab8ece747fcf4a588538ac8e7a15ab16 2024-12-10T06:15:28,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/0171dbe740d1453f8c991d7a61dae3d9 is 50, key is test_row_0/C:col10/1733811327341/Put/seqid=0 2024-12-10T06:15:28,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742151_1327 (size=12301) 2024-12-10T06:15:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811388645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811388645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811388655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811388656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T06:15:28,848 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/0171dbe740d1453f8c991d7a61dae3d9 2024-12-10T06:15:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/7ee5dc9431bd428888e6c4c548a27cf5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7ee5dc9431bd428888e6c4c548a27cf5 2024-12-10T06:15:28,857 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7ee5dc9431bd428888e6c4c548a27cf5, entries=150, sequenceid=423, filesize=12.0 K 2024-12-10T06:15:28,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/ab8ece747fcf4a588538ac8e7a15ab16 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/ab8ece747fcf4a588538ac8e7a15ab16 2024-12-10T06:15:28,862 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/ab8ece747fcf4a588538ac8e7a15ab16, entries=150, sequenceid=423, filesize=12.0 K 2024-12-10T06:15:28,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/0171dbe740d1453f8c991d7a61dae3d9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0171dbe740d1453f8c991d7a61dae3d9 2024-12-10T06:15:28,868 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0171dbe740d1453f8c991d7a61dae3d9, entries=150, sequenceid=423, filesize=12.0 K 2024-12-10T06:15:28,869 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 062b93d61f64862ee02e4bde4cb11903 in 911ms, sequenceid=423, compaction requested=true 2024-12-10T06:15:28,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:28,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:28,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-10T06:15:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-10T06:15:28,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-10T06:15:28,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2220 sec 2024-12-10T06:15:28,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.2260 sec 2024-12-10T06:15:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:29,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T06:15:29,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:29,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:29,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:29,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:29,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:29,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:29,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/fffb004141b44e2a94657757f5c90ce0 is 50, key is test_row_0/A:col10/1733811329156/Put/seqid=0 2024-12-10T06:15:29,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742152_1328 (size=12301) 2024-12-10T06:15:29,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811389190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811389190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811389192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811389192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811389305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811389305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811389305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811389305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,491 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-12-10T06:15:29,491 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:29,491 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,491 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,491 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:29,492 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-12-10T06:15:29,492 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:29,492 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,493 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:29,494 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-12-10T06:15:29,494 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:29,494 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,494 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:29,496 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-12-10T06:15:29,496 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:29,496 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,496 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:29,497 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-12-10T06:15:29,497 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:29,497 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,497 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:29,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811389515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811389515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811389515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811389515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/fffb004141b44e2a94657757f5c90ce0 2024-12-10T06:15:29,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/6bbf2ec4f8c84e5899fa4abf7fcead73 is 50, key is test_row_0/B:col10/1733811329156/Put/seqid=0 2024-12-10T06:15:29,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742153_1329 (size=12301) 2024-12-10T06:15:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T06:15:29,796 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:29,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811389825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811389825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811389825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:29,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811389825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:29,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/6bbf2ec4f8c84e5899fa4abf7fcead73 2024-12-10T06:15:30,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/87d2e69c92f1435483e769f526f373e0 is 50, key is test_row_0/C:col10/1733811329156/Put/seqid=0 2024-12-10T06:15:30,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742154_1330 (size=12301) 2024-12-10T06:15:30,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:30,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:30,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46968 deadline: 1733811390335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46944 deadline: 1733811390335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1733811390335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:30,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46896 deadline: 1733811390335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:15:30,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/87d2e69c92f1435483e769f526f373e0 2024-12-10T06:15:30,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/fffb004141b44e2a94657757f5c90ce0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/fffb004141b44e2a94657757f5c90ce0 2024-12-10T06:15:30,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/fffb004141b44e2a94657757f5c90ce0, entries=150, sequenceid=440, filesize=12.0 K 2024-12-10T06:15:30,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/6bbf2ec4f8c84e5899fa4abf7fcead73 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6bbf2ec4f8c84e5899fa4abf7fcead73 2024-12-10T06:15:30,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6bbf2ec4f8c84e5899fa4abf7fcead73, entries=150, sequenceid=440, filesize=12.0 K 2024-12-10T06:15:30,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/87d2e69c92f1435483e769f526f373e0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/87d2e69c92f1435483e769f526f373e0 2024-12-10T06:15:30,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/87d2e69c92f1435483e769f526f373e0, entries=150, sequenceid=440, filesize=12.0 K 2024-12-10T06:15:30,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 062b93d61f64862ee02e4bde4cb11903 in 1273ms, sequenceid=440, compaction requested=true 2024-12-10T06:15:30,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:30,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:30,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:30,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:30,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:30,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 062b93d61f64862ee02e4bde4cb11903:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:30,432 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:30,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:30,432 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:30,433 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:30,433 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:30,433 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/A is initiating minor compaction (all files) 2024-12-10T06:15:30,433 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/B is initiating minor compaction (all files) 2024-12-10T06:15:30,433 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/A in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:30,433 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/B in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:30,433 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/60dec58905ce4ec7b374c5ee118c103c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f4d0004bd72841a59d4f3cae6dd2d750, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7ee5dc9431bd428888e6c4c548a27cf5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/fffb004141b44e2a94657757f5c90ce0] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=48.9 K 2024-12-10T06:15:30,433 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/2097cdb9be45462399930448034de2c6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/108a4d32b9c04b52ad7d6802811a33f1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/ab8ece747fcf4a588538ac8e7a15ab16, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6bbf2ec4f8c84e5899fa4abf7fcead73] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=48.9 K 2024-12-10T06:15:30,433 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60dec58905ce4ec7b374c5ee118c103c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733811324987 2024-12-10T06:15:30,433 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2097cdb9be45462399930448034de2c6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733811324987 2024-12-10T06:15:30,433 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4d0004bd72841a59d4f3cae6dd2d750, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733811326156 2024-12-10T06:15:30,434 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 108a4d32b9c04b52ad7d6802811a33f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733811326156 2024-12-10T06:15:30,434 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee5dc9431bd428888e6c4c548a27cf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733811327316 2024-12-10T06:15:30,434 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ab8ece747fcf4a588538ac8e7a15ab16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733811327316 2024-12-10T06:15:30,434 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting fffb004141b44e2a94657757f5c90ce0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1733811328014 2024-12-10T06:15:30,434 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bbf2ec4f8c84e5899fa4abf7fcead73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1733811328014 2024-12-10T06:15:30,443 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:30,444 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:30,444 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/bd57e719e402433dbe14188972cc8e05 is 50, key is test_row_0/B:col10/1733811329156/Put/seqid=0 2024-12-10T06:15:30,445 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/dde3469a9e414cdab88a554b205db9de is 50, key is test_row_0/A:col10/1733811329156/Put/seqid=0 2024-12-10T06:15:30,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742156_1332 (size=13323) 2024-12-10T06:15:30,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742155_1331 (size=13323) 2024-12-10T06:15:30,458 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/dde3469a9e414cdab88a554b205db9de as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/dde3469a9e414cdab88a554b205db9de 2024-12-10T06:15:30,463 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/A of 062b93d61f64862ee02e4bde4cb11903 into dde3469a9e414cdab88a554b205db9de(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:30,463 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:30,463 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/A, priority=12, startTime=1733811330431; duration=0sec 2024-12-10T06:15:30,463 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:30,463 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:A 2024-12-10T06:15:30,463 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:30,464 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:30,464 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 062b93d61f64862ee02e4bde4cb11903/C is initiating minor compaction (all files) 2024-12-10T06:15:30,465 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 062b93d61f64862ee02e4bde4cb11903/C in TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:30,465 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/e0abd65a7c914041b460148e5971073f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/425490b8c8d14ff68b3a0a4248086494, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0171dbe740d1453f8c991d7a61dae3d9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/87d2e69c92f1435483e769f526f373e0] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp, totalSize=48.9 K 2024-12-10T06:15:30,465 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0abd65a7c914041b460148e5971073f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733811324987 2024-12-10T06:15:30,465 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 425490b8c8d14ff68b3a0a4248086494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733811326156 2024-12-10T06:15:30,466 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0171dbe740d1453f8c991d7a61dae3d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733811327316 2024-12-10T06:15:30,466 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d2e69c92f1435483e769f526f373e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1733811328014 2024-12-10T06:15:30,474 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 062b93d61f64862ee02e4bde4cb11903#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:30,475 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/a529db11e4d54cc9bbf563212a368cf1 is 50, key is test_row_0/C:col10/1733811329156/Put/seqid=0 2024-12-10T06:15:30,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742157_1333 (size=13323) 2024-12-10T06:15:30,860 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/bd57e719e402433dbe14188972cc8e05 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/bd57e719e402433dbe14188972cc8e05 2024-12-10T06:15:30,865 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/B of 062b93d61f64862ee02e4bde4cb11903 into bd57e719e402433dbe14188972cc8e05(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:30,865 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:30,865 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/B, priority=12, startTime=1733811330432; duration=0sec 2024-12-10T06:15:30,865 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:30,865 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:B 2024-12-10T06:15:30,883 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/a529db11e4d54cc9bbf563212a368cf1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a529db11e4d54cc9bbf563212a368cf1 2024-12-10T06:15:30,888 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 062b93d61f64862ee02e4bde4cb11903/C of 062b93d61f64862ee02e4bde4cb11903 into a529db11e4d54cc9bbf563212a368cf1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:30,888 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:30,888 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., storeName=062b93d61f64862ee02e4bde4cb11903/C, priority=12, startTime=1733811330432; duration=0sec 2024-12-10T06:15:30,888 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:30,888 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 062b93d61f64862ee02e4bde4cb11903:C 2024-12-10T06:15:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:31,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T06:15:31,348 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-12-10T06:15:31,348 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-12-10T06:15:31,348 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:31,348 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:31,348 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,348 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,348 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-12-10T06:15:31,348 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:31,348 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,348 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,348 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:31,348 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:31,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:31,350 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-12-10T06:15:31,350 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:31,350 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:31,350 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:31,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:31,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/8e5a5eae6ab8459faf469241f1c0f701 is 50, key is test_row_0/A:col10/1733811329188/Put/seqid=0 2024-12-10T06:15:31,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742158_1334 (size=12301) 2024-12-10T06:15:31,597 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-12-10T06:15:31,597 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:31,597 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,597 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:31,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T06:15:31,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-10T06:15:31,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-12-10T06:15:31,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-10T06:15:31,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-12-10T06:15:31,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-12-10T06:15:31,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T06:15:31,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2619 2024-12-10T06:15:31,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2515 2024-12-10T06:15:31,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2517 2024-12-10T06:15:31,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2602 2024-12-10T06:15:31,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2513 2024-12-10T06:15:31,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T06:15:31,598 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:15:31,598 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:15:31,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:31,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:31,599 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T06:15:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T06:15:31,602 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811331602"}]},"ts":"1733811331602"} 2024-12-10T06:15:31,603 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T06:15:31,603 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T06:15:31,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T06:15:31,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=062b93d61f64862ee02e4bde4cb11903, UNASSIGN}] 2024-12-10T06:15:31,606 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=062b93d61f64862ee02e4bde4cb11903, UNASSIGN 2024-12-10T06:15:31,607 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=062b93d61f64862ee02e4bde4cb11903, regionState=CLOSING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:31,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=062b93d61f64862ee02e4bde4cb11903, UNASSIGN because future has completed 2024-12-10T06:15:31,609 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:15:31,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:15:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T06:15:31,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/8e5a5eae6ab8459faf469241f1c0f701 2024-12-10T06:15:31,761 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:31,761 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:15:31,761 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 062b93d61f64862ee02e4bde4cb11903, disabling compactions & flushes 2024-12-10T06:15:31,761 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:31,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/21edb517b89d420988a2bdb2c34960cd is 50, key is test_row_0/B:col10/1733811329188/Put/seqid=0 2024-12-10T06:15:31,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742159_1335 (size=12301) 2024-12-10T06:15:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T06:15:32,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/21edb517b89d420988a2bdb2c34960cd 2024-12-10T06:15:32,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c39fbcc0e8b6490482ae76e418d37de1 is 50, key is test_row_0/C:col10/1733811329188/Put/seqid=0 2024-12-10T06:15:32,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742160_1336 (size=12301) 2024-12-10T06:15:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T06:15:32,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c39fbcc0e8b6490482ae76e418d37de1 2024-12-10T06:15:32,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/8e5a5eae6ab8459faf469241f1c0f701 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/8e5a5eae6ab8459faf469241f1c0f701 2024-12-10T06:15:32,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/8e5a5eae6ab8459faf469241f1c0f701, entries=150, sequenceid=462, filesize=12.0 K 2024-12-10T06:15:32,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/21edb517b89d420988a2bdb2c34960cd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/21edb517b89d420988a2bdb2c34960cd 2024-12-10T06:15:32,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/21edb517b89d420988a2bdb2c34960cd, entries=150, sequenceid=462, filesize=12.0 K 2024-12-10T06:15:32,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/c39fbcc0e8b6490482ae76e418d37de1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c39fbcc0e8b6490482ae76e418d37de1 2024-12-10T06:15:32,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c39fbcc0e8b6490482ae76e418d37de1, entries=150, sequenceid=462, filesize=12.0 K 2024-12-10T06:15:32,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for 062b93d61f64862ee02e4bde4cb11903 in 1251ms, sequenceid=462, compaction requested=false 2024-12-10T06:15:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 062b93d61f64862ee02e4bde4cb11903: 2024-12-10T06:15:32,599 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:32,599 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:32,600 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. after waiting 0 ms 2024-12-10T06:15:32,600 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:32,600 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing 062b93d61f64862ee02e4bde4cb11903 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-10T06:15:32,600 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=A 2024-12-10T06:15:32,600 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:32,600 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=B 2024-12-10T06:15:32,600 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:32,600 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 062b93d61f64862ee02e4bde4cb11903, store=C 2024-12-10T06:15:32,600 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:32,603 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/62741924257948ee9302e8bbec49e36b is 50, key is test_row_2/A:col10/1733811331595/Put/seqid=0 2024-12-10T06:15:32,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742161_1337 (size=7415) 2024-12-10T06:15:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T06:15:33,006 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/62741924257948ee9302e8bbec49e36b 2024-12-10T06:15:33,013 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/48e40417761b41fab94c7a958738f4c9 is 50, key is test_row_2/B:col10/1733811331595/Put/seqid=0 2024-12-10T06:15:33,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742162_1338 (size=7415) 2024-12-10T06:15:33,417 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/48e40417761b41fab94c7a958738f4c9 2024-12-10T06:15:33,423 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/4ca8f5101b6840809c47a658f190a791 is 50, key is test_row_2/C:col10/1733811331595/Put/seqid=0 2024-12-10T06:15:33,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742163_1339 (size=7415) 2024-12-10T06:15:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T06:15:33,828 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/4ca8f5101b6840809c47a658f190a791 2024-12-10T06:15:33,832 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/A/62741924257948ee9302e8bbec49e36b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/62741924257948ee9302e8bbec49e36b 2024-12-10T06:15:33,836 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/62741924257948ee9302e8bbec49e36b, entries=50, sequenceid=466, filesize=7.2 K 2024-12-10T06:15:33,837 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/B/48e40417761b41fab94c7a958738f4c9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/48e40417761b41fab94c7a958738f4c9 2024-12-10T06:15:33,841 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/48e40417761b41fab94c7a958738f4c9, entries=50, sequenceid=466, filesize=7.2 K 2024-12-10T06:15:33,842 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/.tmp/C/4ca8f5101b6840809c47a658f190a791 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/4ca8f5101b6840809c47a658f190a791 2024-12-10T06:15:33,845 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/4ca8f5101b6840809c47a658f190a791, entries=50, sequenceid=466, filesize=7.2 K 2024-12-10T06:15:33,846 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 062b93d61f64862ee02e4bde4cb11903 in 1246ms, sequenceid=466, compaction requested=true 2024-12-10T06:15:33,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c824db809fd245be9c07c80aafe5f9af, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/e7198e362871452ca0a083c72b54f8b0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7da0c999bdd348deab3cc0fd5e3083d1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7458f389d76a4370867456119d300b0b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bc93688ced2e4061b0a636baad186a01, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a20bf6c5808949018e3b80add166af03, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ccc1b289fdac4c8bab8d4b17d2235421, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/24f9d7704b98498984ec426b89ce23f1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/dfbb475fb1c64dac8db841242d5e98af, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/73d12e41c49d474aa9b31e0031e08eb4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c91951409c5846348a394789495934e0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a88bace5501d4937bb86828839116fcf, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/eae0f46bc6974b7ba7e1acddb5974ee5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a0f3eeb11f634bc486131069aaa665a9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/5057a3b3c40b4e03ad97fe1706ce426d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/01cfce04811d40e49d8c7ba15c7e5bcf, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/315a67c2dc1543ebacd492a6cc754bf3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/9e27761b4e5f4b128727066e6c18d90d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f9db83632e8840d285a4e16ad4a20e1d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/cd25f0ccf920408981c2c6b8f49ba427, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/63c6d9cc9e584aff915557e3d3c2c8dc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bdb1c069731547c0a1bfaa8e7483b24e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7c37d943373149f2972fbd560688a230, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c019e6caecef4476a6b4692ee758dd56, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/3edccb92399c4ab488e9c9446a6f0c3b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/60dec58905ce4ec7b374c5ee118c103c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ce983e93e3ff423d94747d43be00fa91, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f4d0004bd72841a59d4f3cae6dd2d750, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7ee5dc9431bd428888e6c4c548a27cf5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/fffb004141b44e2a94657757f5c90ce0] to archive 2024-12-10T06:15:33,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:15:33,851 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7458f389d76a4370867456119d300b0b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7458f389d76a4370867456119d300b0b 2024-12-10T06:15:33,851 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/24f9d7704b98498984ec426b89ce23f1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/24f9d7704b98498984ec426b89ce23f1 2024-12-10T06:15:33,851 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ccc1b289fdac4c8bab8d4b17d2235421 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ccc1b289fdac4c8bab8d4b17d2235421 2024-12-10T06:15:33,851 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c824db809fd245be9c07c80aafe5f9af to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c824db809fd245be9c07c80aafe5f9af 2024-12-10T06:15:33,852 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a20bf6c5808949018e3b80add166af03 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a20bf6c5808949018e3b80add166af03 2024-12-10T06:15:33,852 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/e7198e362871452ca0a083c72b54f8b0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/e7198e362871452ca0a083c72b54f8b0 2024-12-10T06:15:33,852 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7da0c999bdd348deab3cc0fd5e3083d1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7da0c999bdd348deab3cc0fd5e3083d1 2024-12-10T06:15:33,852 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bc93688ced2e4061b0a636baad186a01 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bc93688ced2e4061b0a636baad186a01 2024-12-10T06:15:33,853 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a88bace5501d4937bb86828839116fcf to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a88bace5501d4937bb86828839116fcf 2024-12-10T06:15:33,853 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/73d12e41c49d474aa9b31e0031e08eb4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/73d12e41c49d474aa9b31e0031e08eb4 2024-12-10T06:15:33,854 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/dfbb475fb1c64dac8db841242d5e98af to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/dfbb475fb1c64dac8db841242d5e98af 2024-12-10T06:15:33,854 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c91951409c5846348a394789495934e0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c91951409c5846348a394789495934e0 2024-12-10T06:15:33,854 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/eae0f46bc6974b7ba7e1acddb5974ee5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/eae0f46bc6974b7ba7e1acddb5974ee5 2024-12-10T06:15:33,855 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/01cfce04811d40e49d8c7ba15c7e5bcf to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/01cfce04811d40e49d8c7ba15c7e5bcf 2024-12-10T06:15:33,855 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/5057a3b3c40b4e03ad97fe1706ce426d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/5057a3b3c40b4e03ad97fe1706ce426d 2024-12-10T06:15:33,855 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a0f3eeb11f634bc486131069aaa665a9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/a0f3eeb11f634bc486131069aaa665a9 2024-12-10T06:15:33,856 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/9e27761b4e5f4b128727066e6c18d90d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/9e27761b4e5f4b128727066e6c18d90d 2024-12-10T06:15:33,857 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f9db83632e8840d285a4e16ad4a20e1d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f9db83632e8840d285a4e16ad4a20e1d 2024-12-10T06:15:33,857 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/cd25f0ccf920408981c2c6b8f49ba427 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/cd25f0ccf920408981c2c6b8f49ba427 2024-12-10T06:15:33,857 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/315a67c2dc1543ebacd492a6cc754bf3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/315a67c2dc1543ebacd492a6cc754bf3 2024-12-10T06:15:33,858 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7c37d943373149f2972fbd560688a230 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7c37d943373149f2972fbd560688a230 2024-12-10T06:15:33,858 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/63c6d9cc9e584aff915557e3d3c2c8dc to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/63c6d9cc9e584aff915557e3d3c2c8dc 2024-12-10T06:15:33,858 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bdb1c069731547c0a1bfaa8e7483b24e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/bdb1c069731547c0a1bfaa8e7483b24e 2024-12-10T06:15:33,858 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c019e6caecef4476a6b4692ee758dd56 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/c019e6caecef4476a6b4692ee758dd56 2024-12-10T06:15:33,858 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/3edccb92399c4ab488e9c9446a6f0c3b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/3edccb92399c4ab488e9c9446a6f0c3b 2024-12-10T06:15:33,859 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/60dec58905ce4ec7b374c5ee118c103c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/60dec58905ce4ec7b374c5ee118c103c 2024-12-10T06:15:33,859 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ce983e93e3ff423d94747d43be00fa91 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/ce983e93e3ff423d94747d43be00fa91 2024-12-10T06:15:33,860 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7ee5dc9431bd428888e6c4c548a27cf5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/7ee5dc9431bd428888e6c4c548a27cf5 2024-12-10T06:15:33,860 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/fffb004141b44e2a94657757f5c90ce0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/fffb004141b44e2a94657757f5c90ce0 2024-12-10T06:15:33,860 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f4d0004bd72841a59d4f3cae6dd2d750 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/f4d0004bd72841a59d4f3cae6dd2d750 2024-12-10T06:15:33,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/062a5762a75b4d49b11faa969959a840, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a280a2ce5a674ea3a5e381517a27363f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/e3c8a46525b24de08b3ec41325ae3a84, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/d4997c2ff6c74eff8f46ff7df476e6ab, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/340d182b51e04c8d872cd3572659bde8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/fde44ca76b8e44c7ac73ebc5fd3a849d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/5d84f7e638a1470a8b3ad27a5e914c01, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/f9f4528815194c99b28e7e61c455506c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/4fa223826cc34dba806ca1c3ad12e40f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/bb0e250b4af045a0b0c57a5f0fc19c24, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/726f0eeb1c2f4e4bbc3707003a5af1b5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/56db2896b9fe4c4ab52fa026dea4ae77, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/20a8b18b71654065b963abfbed9954c6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/b61219cd561d486892deebf133eb468b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/7b0e6af298f7487c8403bd714a3711b3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/15734191d62c43d189815562ba8b45e2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/90f7d20fab09400a985c4894d2b21564, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/9af021563bec4f04b4f17ef2b7ed4df6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a5e2de423fc94fed9bf7df201f799de7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6f02327ad57d4ac496407631706e04cc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/3934de8299d14fdb85df51503b6e8de2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/753ff9e8865a43feab978137e3e67b58, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/1198b030d50f4c3ebf48dc13f783ebfc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/17be7fa1abf947bab51f9e588e4243a1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/c91a964177604e4e82896e77a410c176, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/2097cdb9be45462399930448034de2c6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/86e247bf38a349ce99a5b700368d6cc6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/108a4d32b9c04b52ad7d6802811a33f1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/ab8ece747fcf4a588538ac8e7a15ab16, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6bbf2ec4f8c84e5899fa4abf7fcead73] to archive 2024-12-10T06:15:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:15:33,864 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/d4997c2ff6c74eff8f46ff7df476e6ab to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/d4997c2ff6c74eff8f46ff7df476e6ab 2024-12-10T06:15:33,864 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a280a2ce5a674ea3a5e381517a27363f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a280a2ce5a674ea3a5e381517a27363f 2024-12-10T06:15:33,864 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/340d182b51e04c8d872cd3572659bde8 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/340d182b51e04c8d872cd3572659bde8 2024-12-10T06:15:33,864 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/f9f4528815194c99b28e7e61c455506c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/f9f4528815194c99b28e7e61c455506c 2024-12-10T06:15:33,865 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/062a5762a75b4d49b11faa969959a840 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/062a5762a75b4d49b11faa969959a840 2024-12-10T06:15:33,865 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/e3c8a46525b24de08b3ec41325ae3a84 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/e3c8a46525b24de08b3ec41325ae3a84 2024-12-10T06:15:33,865 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/fde44ca76b8e44c7ac73ebc5fd3a849d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/fde44ca76b8e44c7ac73ebc5fd3a849d 2024-12-10T06:15:33,865 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/5d84f7e638a1470a8b3ad27a5e914c01 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/5d84f7e638a1470a8b3ad27a5e914c01 2024-12-10T06:15:33,866 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/bb0e250b4af045a0b0c57a5f0fc19c24 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/bb0e250b4af045a0b0c57a5f0fc19c24 2024-12-10T06:15:33,866 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/56db2896b9fe4c4ab52fa026dea4ae77 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/56db2896b9fe4c4ab52fa026dea4ae77 2024-12-10T06:15:33,867 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/20a8b18b71654065b963abfbed9954c6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/20a8b18b71654065b963abfbed9954c6 2024-12-10T06:15:33,867 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/726f0eeb1c2f4e4bbc3707003a5af1b5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/726f0eeb1c2f4e4bbc3707003a5af1b5 2024-12-10T06:15:33,867 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/b61219cd561d486892deebf133eb468b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/b61219cd561d486892deebf133eb468b 2024-12-10T06:15:33,868 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/15734191d62c43d189815562ba8b45e2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/15734191d62c43d189815562ba8b45e2 2024-12-10T06:15:33,868 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/90f7d20fab09400a985c4894d2b21564 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/90f7d20fab09400a985c4894d2b21564 2024-12-10T06:15:33,869 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a5e2de423fc94fed9bf7df201f799de7 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/a5e2de423fc94fed9bf7df201f799de7 2024-12-10T06:15:33,869 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/9af021563bec4f04b4f17ef2b7ed4df6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/9af021563bec4f04b4f17ef2b7ed4df6 2024-12-10T06:15:33,869 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/3934de8299d14fdb85df51503b6e8de2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/3934de8299d14fdb85df51503b6e8de2 2024-12-10T06:15:33,870 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6f02327ad57d4ac496407631706e04cc to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6f02327ad57d4ac496407631706e04cc 2024-12-10T06:15:33,870 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/7b0e6af298f7487c8403bd714a3711b3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/7b0e6af298f7487c8403bd714a3711b3 2024-12-10T06:15:33,870 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/4fa223826cc34dba806ca1c3ad12e40f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/4fa223826cc34dba806ca1c3ad12e40f 2024-12-10T06:15:33,870 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/1198b030d50f4c3ebf48dc13f783ebfc to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/1198b030d50f4c3ebf48dc13f783ebfc 2024-12-10T06:15:33,870 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/753ff9e8865a43feab978137e3e67b58 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/753ff9e8865a43feab978137e3e67b58 2024-12-10T06:15:33,871 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/17be7fa1abf947bab51f9e588e4243a1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/17be7fa1abf947bab51f9e588e4243a1 2024-12-10T06:15:33,871 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/c91a964177604e4e82896e77a410c176 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/c91a964177604e4e82896e77a410c176 2024-12-10T06:15:33,871 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/2097cdb9be45462399930448034de2c6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/2097cdb9be45462399930448034de2c6 2024-12-10T06:15:33,871 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/86e247bf38a349ce99a5b700368d6cc6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/86e247bf38a349ce99a5b700368d6cc6 2024-12-10T06:15:33,871 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/108a4d32b9c04b52ad7d6802811a33f1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/108a4d32b9c04b52ad7d6802811a33f1 2024-12-10T06:15:33,872 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6bbf2ec4f8c84e5899fa4abf7fcead73 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/6bbf2ec4f8c84e5899fa4abf7fcead73 2024-12-10T06:15:33,872 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/ab8ece747fcf4a588538ac8e7a15ab16 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/ab8ece747fcf4a588538ac8e7a15ab16 2024-12-10T06:15:33,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/3c1420b027684ead8541f6d537d2b9ef, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c20ec0bbd1494329a35897a271fdc74e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/117ca9c04c9e44bca1aa712818257e24, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/82ed9802f3b141b48b9d4561125dd8c4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/60c855ab04ad4898b506a317a7278ee0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0583ede398bf47fc850c7c23befb44e1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/f164b9b2da1243c2abfdd8a31a1c94f4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0b22f652dd2e44a69b767e3754ed343d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/48547ec1758c4bffb2ee17f42f134b3d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/b5dc57d5157f4670ad939450a88d5315, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a1ea4956df4847b6bf1c6597206fbdec, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/479a9a0ad7694321bb8782a9996662b5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/dcf21ac5b25e4c8a832ff67fce06da44, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/32f7439ae4fd452398f35211dba39279, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ae0f747bad6a4126b2477940cc625156, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/8e99ec692d42430c8df0d41e4dc7c854, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/64f0f72f70c9413e9cf15b94b296cf36, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/1d1a7753e6a94b8098742fc7cc443889, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/30e37925d6e540cda62ce3a0c4250074, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c6e942db0f57456e87bf84145365c754, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a65ec64e79ef400cb26e264dc9aaaf77, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/720b0c3792084cb58ddd7ca3eb04b1e9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/2950d0a6f9f54efea079cabe1e058616, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c990af843f8b4f5cbe4bbd6bab7f129b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/91d0fb88f5c54be4b3a56ebb7f36060b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/e0abd65a7c914041b460148e5971073f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ff1cc3dcaf42445eba7d88f7c142ce09, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/425490b8c8d14ff68b3a0a4248086494, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0171dbe740d1453f8c991d7a61dae3d9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/87d2e69c92f1435483e769f526f373e0] to archive 2024-12-10T06:15:33,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:15:33,876 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c20ec0bbd1494329a35897a271fdc74e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c20ec0bbd1494329a35897a271fdc74e 2024-12-10T06:15:33,876 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/82ed9802f3b141b48b9d4561125dd8c4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/82ed9802f3b141b48b9d4561125dd8c4 2024-12-10T06:15:33,876 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/3c1420b027684ead8541f6d537d2b9ef to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/3c1420b027684ead8541f6d537d2b9ef 2024-12-10T06:15:33,876 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/117ca9c04c9e44bca1aa712818257e24 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/117ca9c04c9e44bca1aa712818257e24 2024-12-10T06:15:33,876 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/60c855ab04ad4898b506a317a7278ee0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/60c855ab04ad4898b506a317a7278ee0 2024-12-10T06:15:33,876 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0583ede398bf47fc850c7c23befb44e1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0583ede398bf47fc850c7c23befb44e1 2024-12-10T06:15:33,877 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0b22f652dd2e44a69b767e3754ed343d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0b22f652dd2e44a69b767e3754ed343d 2024-12-10T06:15:33,877 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/f164b9b2da1243c2abfdd8a31a1c94f4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/f164b9b2da1243c2abfdd8a31a1c94f4 2024-12-10T06:15:33,877 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/b5dc57d5157f4670ad939450a88d5315 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/b5dc57d5157f4670ad939450a88d5315 2024-12-10T06:15:33,877 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a1ea4956df4847b6bf1c6597206fbdec to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a1ea4956df4847b6bf1c6597206fbdec 2024-12-10T06:15:33,878 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/479a9a0ad7694321bb8782a9996662b5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/479a9a0ad7694321bb8782a9996662b5 2024-12-10T06:15:33,878 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/48547ec1758c4bffb2ee17f42f134b3d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/48547ec1758c4bffb2ee17f42f134b3d 2024-12-10T06:15:33,878 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/dcf21ac5b25e4c8a832ff67fce06da44 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/dcf21ac5b25e4c8a832ff67fce06da44 2024-12-10T06:15:33,878 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/32f7439ae4fd452398f35211dba39279 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/32f7439ae4fd452398f35211dba39279 2024-12-10T06:15:33,879 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ae0f747bad6a4126b2477940cc625156 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ae0f747bad6a4126b2477940cc625156 2024-12-10T06:15:33,879 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/8e99ec692d42430c8df0d41e4dc7c854 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/8e99ec692d42430c8df0d41e4dc7c854 2024-12-10T06:15:33,879 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/64f0f72f70c9413e9cf15b94b296cf36 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/64f0f72f70c9413e9cf15b94b296cf36 2024-12-10T06:15:33,880 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/1d1a7753e6a94b8098742fc7cc443889 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/1d1a7753e6a94b8098742fc7cc443889 2024-12-10T06:15:33,880 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/30e37925d6e540cda62ce3a0c4250074 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/30e37925d6e540cda62ce3a0c4250074 2024-12-10T06:15:33,880 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a65ec64e79ef400cb26e264dc9aaaf77 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a65ec64e79ef400cb26e264dc9aaaf77 2024-12-10T06:15:33,880 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c6e942db0f57456e87bf84145365c754 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c6e942db0f57456e87bf84145365c754 2024-12-10T06:15:33,881 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/720b0c3792084cb58ddd7ca3eb04b1e9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/720b0c3792084cb58ddd7ca3eb04b1e9 2024-12-10T06:15:33,881 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/2950d0a6f9f54efea079cabe1e058616 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/2950d0a6f9f54efea079cabe1e058616 2024-12-10T06:15:33,881 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c990af843f8b4f5cbe4bbd6bab7f129b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c990af843f8b4f5cbe4bbd6bab7f129b 2024-12-10T06:15:33,881 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/91d0fb88f5c54be4b3a56ebb7f36060b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/91d0fb88f5c54be4b3a56ebb7f36060b 2024-12-10T06:15:33,881 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/e0abd65a7c914041b460148e5971073f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/e0abd65a7c914041b460148e5971073f 2024-12-10T06:15:33,881 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ff1cc3dcaf42445eba7d88f7c142ce09 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/ff1cc3dcaf42445eba7d88f7c142ce09 2024-12-10T06:15:33,882 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/425490b8c8d14ff68b3a0a4248086494 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/425490b8c8d14ff68b3a0a4248086494 2024-12-10T06:15:33,882 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0171dbe740d1453f8c991d7a61dae3d9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/0171dbe740d1453f8c991d7a61dae3d9 2024-12-10T06:15:33,882 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/87d2e69c92f1435483e769f526f373e0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/87d2e69c92f1435483e769f526f373e0 2024-12-10T06:15:33,886 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/recovered.edits/469.seqid, newMaxSeqId=469, maxSeqId=1 2024-12-10T06:15:33,887 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. 2024-12-10T06:15:33,887 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 062b93d61f64862ee02e4bde4cb11903: Waiting for close lock at 1733811331761Running coprocessor pre-close hooks at 1733811331761Disabling compacts and flushes for region at 1733811331761Disabling writes for close at 1733811332600 (+839 ms)Obtaining lock to block concurrent updates at 1733811332600Preparing flush snapshotting stores in 062b93d61f64862ee02e4bde4cb11903 at 1733811332600Finished memstore snapshotting TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1733811332600Flushing stores of TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903. at 1733811332600Flushing 062b93d61f64862ee02e4bde4cb11903/A: creating writer at 1733811332600Flushing 062b93d61f64862ee02e4bde4cb11903/A: appending metadata at 1733811332602 (+2 ms)Flushing 062b93d61f64862ee02e4bde4cb11903/A: closing flushed file at 1733811332602Flushing 062b93d61f64862ee02e4bde4cb11903/B: creating writer at 1733811333010 (+408 ms)Flushing 062b93d61f64862ee02e4bde4cb11903/B: appending metadata at 1733811333012 (+2 ms)Flushing 062b93d61f64862ee02e4bde4cb11903/B: closing flushed file at 1733811333012Flushing 062b93d61f64862ee02e4bde4cb11903/C: creating writer at 1733811333421 (+409 ms)Flushing 062b93d61f64862ee02e4bde4cb11903/C: appending metadata at 1733811333423 (+2 ms)Flushing 062b93d61f64862ee02e4bde4cb11903/C: closing flushed file at 1733811333423Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c350ab2: reopening flushed file at 1733811333831 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40efc641: reopening flushed file at 1733811333836 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16353173: reopening flushed file at 1733811333841 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 062b93d61f64862ee02e4bde4cb11903 in 1246ms, sequenceid=466, compaction requested=true at 1733811333846 (+5 ms)Writing region close event to WAL at 1733811333883 (+37 ms)Running coprocessor post-close hooks at 1733811333887 (+4 ms)Closed at 1733811333887 2024-12-10T06:15:33,889 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:33,890 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=062b93d61f64862ee02e4bde4cb11903, regionState=CLOSED 2024-12-10T06:15:33,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:15:33,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-10T06:15:33,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 062b93d61f64862ee02e4bde4cb11903, server=c42c682b34f7,43983,1733811247584 in 2.2830 sec 2024-12-10T06:15:33,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-10T06:15:33,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=062b93d61f64862ee02e4bde4cb11903, UNASSIGN in 2.2890 sec 2024-12-10T06:15:33,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-10T06:15:33,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2920 sec 2024-12-10T06:15:33,898 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811333898"}]},"ts":"1733811333898"} 2024-12-10T06:15:33,900 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T06:15:33,900 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T06:15:33,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3010 sec 2024-12-10T06:15:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T06:15:35,766 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:35,766 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T06:15:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:35,768 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-10T06:15:35,769 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:35,771 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:35,773 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/recovered.edits] 2024-12-10T06:15:35,776 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/8e5a5eae6ab8459faf469241f1c0f701 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/8e5a5eae6ab8459faf469241f1c0f701 2024-12-10T06:15:35,776 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/62741924257948ee9302e8bbec49e36b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/62741924257948ee9302e8bbec49e36b 2024-12-10T06:15:35,776 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/dde3469a9e414cdab88a554b205db9de to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/A/dde3469a9e414cdab88a554b205db9de 2024-12-10T06:15:35,779 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/48e40417761b41fab94c7a958738f4c9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/48e40417761b41fab94c7a958738f4c9 2024-12-10T06:15:35,779 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/21edb517b89d420988a2bdb2c34960cd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/21edb517b89d420988a2bdb2c34960cd 2024-12-10T06:15:35,779 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/bd57e719e402433dbe14188972cc8e05 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/B/bd57e719e402433dbe14188972cc8e05 2024-12-10T06:15:35,781 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a529db11e4d54cc9bbf563212a368cf1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/a529db11e4d54cc9bbf563212a368cf1 2024-12-10T06:15:35,781 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/4ca8f5101b6840809c47a658f190a791 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/4ca8f5101b6840809c47a658f190a791 2024-12-10T06:15:35,781 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c39fbcc0e8b6490482ae76e418d37de1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/C/c39fbcc0e8b6490482ae76e418d37de1 2024-12-10T06:15:35,784 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/recovered.edits/469.seqid to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903/recovered.edits/469.seqid 2024-12-10T06:15:35,784 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/062b93d61f64862ee02e4bde4cb11903 2024-12-10T06:15:35,785 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T06:15:35,786 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:35,789 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T06:15:35,791 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T06:15:35,793 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:35,793 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T06:15:35,793 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733811335793"}]},"ts":"9223372036854775807"} 2024-12-10T06:15:35,794 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T06:15:35,794 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 062b93d61f64862ee02e4bde4cb11903, NAME => 'TestAcidGuarantees,,1733811307197.062b93d61f64862ee02e4bde4cb11903.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T06:15:35,794 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T06:15:35,795 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733811335794"}]},"ts":"9223372036854775807"} 2024-12-10T06:15:35,796 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T06:15:35,797 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:35,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-12-10T06:15:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-10T06:15:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T06:15:35,876 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:35,885 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=256 (was 258), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=479 (was 473) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6603 (was 7291) 2024-12-10T06:15:35,894 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=256, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=479, ProcessCount=11, AvailableMemoryMB=6602 2024-12-10T06:15:35,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T06:15:35,896 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:15:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:35,897 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T06:15:35,897 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:35,898 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-12-10T06:15:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T06:15:35,898 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T06:15:35,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742164_1340 (size=1061) 2024-12-10T06:15:35,915 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T06:15:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T06:15:36,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T06:15:36,305 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6 2024-12-10T06:15:36,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742165_1341 (size=53) 2024-12-10T06:15:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T06:15:36,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:15:36,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1c35e623ec588c4a8cce9c8097008587, disabling compactions & flushes 2024-12-10T06:15:36,711 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:36,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:36,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. after waiting 0 ms 2024-12-10T06:15:36,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:36,711 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:36,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1c35e623ec588c4a8cce9c8097008587: Waiting for close lock at 1733811336711Disabling compacts and flushes for region at 1733811336711Disabling writes for close at 1733811336711Writing region close event to WAL at 1733811336711Closed at 1733811336711 2024-12-10T06:15:36,712 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T06:15:36,712 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733811336712"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733811336712"}]},"ts":"1733811336712"} 2024-12-10T06:15:36,714 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T06:15:36,715 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T06:15:36,715 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811336715"}]},"ts":"1733811336715"} 2024-12-10T06:15:36,716 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T06:15:36,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, ASSIGN}] 2024-12-10T06:15:36,718 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, ASSIGN 2024-12-10T06:15:36,718 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, ASSIGN; state=OFFLINE, location=c42c682b34f7,43983,1733811247584; forceNewPlan=false, retain=false 2024-12-10T06:15:36,869 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=1c35e623ec588c4a8cce9c8097008587, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:36,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, ASSIGN because future has completed 2024-12-10T06:15:36,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:15:37,025 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:37,026 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:15:37,026 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,026 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:15:37,026 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,026 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,027 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,028 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:15:37,029 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c35e623ec588c4a8cce9c8097008587 columnFamilyName A 2024-12-10T06:15:37,029 DEBUG [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:37,029 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(327): Store=1c35e623ec588c4a8cce9c8097008587/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:15:37,029 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,030 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:15:37,030 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c35e623ec588c4a8cce9c8097008587 columnFamilyName B 2024-12-10T06:15:37,030 DEBUG [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:37,031 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(327): Store=1c35e623ec588c4a8cce9c8097008587/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:15:37,031 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,031 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:15:37,031 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c35e623ec588c4a8cce9c8097008587 columnFamilyName C 2024-12-10T06:15:37,032 DEBUG [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:37,032 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(327): Store=1c35e623ec588c4a8cce9c8097008587/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:15:37,032 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:37,032 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,033 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,033 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,034 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,034 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,034 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:15:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T06:15:37,035 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,037 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:15:37,037 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 1c35e623ec588c4a8cce9c8097008587; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72525591, jitterRate=0.08071552217006683}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:15:37,037 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:37,038 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 1c35e623ec588c4a8cce9c8097008587: Running coprocessor pre-open hook at 1733811337026Writing region info on filesystem at 1733811337026Initializing all the Stores at 1733811337027 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811337027Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811337027Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811337027Cleaning up temporary data from old regions at 1733811337034 (+7 ms)Running coprocessor post-open hooks at 1733811337037 (+3 ms)Region opened successfully at 1733811337038 (+1 ms) 2024-12-10T06:15:37,038 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., pid=83, masterSystemTime=1733811337022 2024-12-10T06:15:37,040 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:37,040 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:37,041 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=1c35e623ec588c4a8cce9c8097008587, regionState=OPEN, openSeqNum=2, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:37,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:15:37,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-10T06:15:37,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 in 172 msec 2024-12-10T06:15:37,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-10T06:15:37,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, ASSIGN in 327 msec 2024-12-10T06:15:37,046 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T06:15:37,046 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811337046"}]},"ts":"1733811337046"} 2024-12-10T06:15:37,047 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T06:15:37,048 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T06:15:37,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-12-10T06:15:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T06:15:38,045 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:38,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11317fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,047 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2a80c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,049 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4699108c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,049 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:38,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:38,051 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T06:15:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c42c682b34f7,37415,1733811246834 2024-12-10T06:15:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f98b82a 2024-12-10T06:15:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T06:15:38,053 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T06:15:38,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T06:15:38,054 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:15:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T06:15:38,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742166_1342 (size=1097) 2024-12-10T06:15:38,464 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T06:15:38,465 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T06:15:38,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T06:15:38,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, REOPEN/MOVE}] 2024-12-10T06:15:38,469 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, REOPEN/MOVE 2024-12-10T06:15:38,470 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=1c35e623ec588c4a8cce9c8097008587, regionState=CLOSING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:38,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, REOPEN/MOVE because future has completed 2024-12-10T06:15:38,471 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:15:38,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:15:38,624 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,624 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:15:38,624 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 1c35e623ec588c4a8cce9c8097008587, disabling compactions & flushes 2024-12-10T06:15:38,624 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:38,624 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:38,624 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. after waiting 0 ms 2024-12-10T06:15:38,624 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:38,627 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T06:15:38,627 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:38,628 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 1c35e623ec588c4a8cce9c8097008587: Waiting for close lock at 1733811338624Running coprocessor pre-close hooks at 1733811338624Disabling compacts and flushes for region at 1733811338624Disabling writes for close at 1733811338624Writing region close event to WAL at 1733811338624Running coprocessor post-close hooks at 1733811338627 (+3 ms)Closed at 1733811338627 2024-12-10T06:15:38,628 WARN [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 1c35e623ec588c4a8cce9c8097008587 to self. 2024-12-10T06:15:38,629 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,630 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=1c35e623ec588c4a8cce9c8097008587, regionState=CLOSED 2024-12-10T06:15:38,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:15:38,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-10T06:15:38,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 in 161 msec 2024-12-10T06:15:38,633 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, REOPEN/MOVE; state=CLOSED, location=c42c682b34f7,43983,1733811247584; forceNewPlan=false, retain=true 2024-12-10T06:15:38,784 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=1c35e623ec588c4a8cce9c8097008587, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:38,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, REOPEN/MOVE because future has completed 2024-12-10T06:15:38,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:15:38,940 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:38,940 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:15:38,940 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,940 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:15:38,940 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,940 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,941 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,942 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:15:38,943 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c35e623ec588c4a8cce9c8097008587 columnFamilyName A 2024-12-10T06:15:38,944 DEBUG [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:38,944 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(327): Store=1c35e623ec588c4a8cce9c8097008587/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:15:38,944 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,945 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:15:38,945 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c35e623ec588c4a8cce9c8097008587 columnFamilyName B 2024-12-10T06:15:38,945 DEBUG [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:38,945 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(327): Store=1c35e623ec588c4a8cce9c8097008587/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:15:38,945 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,946 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:15:38,946 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c35e623ec588c4a8cce9c8097008587 columnFamilyName C 2024-12-10T06:15:38,946 DEBUG [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:38,946 INFO [StoreOpener-1c35e623ec588c4a8cce9c8097008587-1 {}] regionserver.HStore(327): Store=1c35e623ec588c4a8cce9c8097008587/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:15:38,946 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:38,946 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,947 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,948 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,948 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,948 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,949 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:15:38,950 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,950 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 1c35e623ec588c4a8cce9c8097008587; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70660841, jitterRate=0.052928581833839417}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:15:38,950 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:38,951 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 1c35e623ec588c4a8cce9c8097008587: Running coprocessor pre-open hook at 1733811338941Writing region info on filesystem at 1733811338941Initializing all the Stores at 1733811338941Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811338941Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811338941Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811338941Cleaning up temporary data from old regions at 1733811338948 (+7 ms)Running coprocessor post-open hooks at 1733811338950 (+2 ms)Region opened successfully at 1733811338951 (+1 ms) 2024-12-10T06:15:38,951 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., pid=88, masterSystemTime=1733811338937 2024-12-10T06:15:38,953 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:38,953 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:38,953 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=1c35e623ec588c4a8cce9c8097008587, regionState=OPEN, openSeqNum=5, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:38,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:15:38,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-12-10T06:15:38,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 in 169 msec 2024-12-10T06:15:38,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-10T06:15:38,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, REOPEN/MOVE in 488 msec 2024-12-10T06:15:38,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-10T06:15:38,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-12-10T06:15:38,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-12-10T06:15:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-10T06:15:38,962 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:38,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,964 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b332893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,965 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ebde18a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,967 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675bafc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,968 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378d5675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,970 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1051064a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310e3323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,972 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ec614c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,973 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40cf04de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,975 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@352c206a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,976 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dcc1aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31af8ab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,979 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@801c0e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,980 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e06f847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc5710b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,983 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4555df52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,984 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511a0f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,986 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ea5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,987 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e3ffa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a84aab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:15:38,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:15:38,989 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:15:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:15:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:15:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66762e25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:15:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:15:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:38,990 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:15:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aef70f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:15:38,991 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,991 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,991 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,991 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,991 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,991 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:38,992 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:38,993 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:38,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:38,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:38,994 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-10T06:15:38,994 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:38,994 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:38,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:38,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:38,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T06:15:38,996 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:38,997 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:38,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:38,999 DEBUG [Thread-1573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:39,000 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:15:39,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:15:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:15:39,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:39,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:15:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:15:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:39,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T06:15:39,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:39,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:39,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:39,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:39,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:39,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:39,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121017e94ea9fde54da48df64238075f209d_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811339009/Put/seqid=0 2024-12-10T06:15:39,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742167_1343 (size=12154) 2024-12-10T06:15:39,053 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:39,057 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121017e94ea9fde54da48df64238075f209d_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121017e94ea9fde54da48df64238075f209d_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:39,058 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/341f0e0eb6a54c4e93c31e5cc07e038c, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:39,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/341f0e0eb6a54c4e93c31e5cc07e038c is 175, key is test_row_0/A:col10/1733811339009/Put/seqid=0 2024-12-10T06:15:39,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742168_1344 (size=30955) 2024-12-10T06:15:39,065 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/341f0e0eb6a54c4e93c31e5cc07e038c 2024-12-10T06:15:39,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811399052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811399055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811399053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811399056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811399071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/f7917a425f1942069726be9438b989f5 is 50, key is test_row_0/B:col10/1733811339009/Put/seqid=0 2024-12-10T06:15:39,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742169_1345 (size=12001) 2024-12-10T06:15:39,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/f7917a425f1942069726be9438b989f5 2024-12-10T06:15:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T06:15:39,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/120d089aba6b46378bfd8e1cc299a9c1 is 50, key is test_row_0/C:col10/1733811339009/Put/seqid=0 2024-12-10T06:15:39,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742170_1346 (size=12001) 2024-12-10T06:15:39,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/120d089aba6b46378bfd8e1cc299a9c1 2024-12-10T06:15:39,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/341f0e0eb6a54c4e93c31e5cc07e038c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/341f0e0eb6a54c4e93c31e5cc07e038c 2024-12-10T06:15:39,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/341f0e0eb6a54c4e93c31e5cc07e038c, entries=150, sequenceid=17, filesize=30.2 K 2024-12-10T06:15:39,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/f7917a425f1942069726be9438b989f5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/f7917a425f1942069726be9438b989f5 2024-12-10T06:15:39,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/f7917a425f1942069726be9438b989f5, entries=150, sequenceid=17, filesize=11.7 K 2024-12-10T06:15:39,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/120d089aba6b46378bfd8e1cc299a9c1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/120d089aba6b46378bfd8e1cc299a9c1 2024-12-10T06:15:39,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/120d089aba6b46378bfd8e1cc299a9c1, entries=150, sequenceid=17, filesize=11.7 K 2024-12-10T06:15:39,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1c35e623ec588c4a8cce9c8097008587 in 132ms, sequenceid=17, compaction requested=false 2024-12-10T06:15:39,145 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T06:15:39,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:39,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T06:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:39,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T06:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107a7ea24c40d94f2784cddc555ff2c3cd_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811339069/Put/seqid=0 2024-12-10T06:15:39,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742171_1347 (size=12154) 2024-12-10T06:15:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:39,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811399195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811399196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811399210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811399212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811399212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T06:15:39,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811399316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811399325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811399335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811399335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,411 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:15:39,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811399415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811399525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811399546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811399555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811399555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:39,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107a7ea24c40d94f2784cddc555ff2c3cd_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107a7ea24c40d94f2784cddc555ff2c3cd_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:39,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/61381710d4024806a47185bcfecd51cb, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:39,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/61381710d4024806a47185bcfecd51cb is 175, key is test_row_0/A:col10/1733811339069/Put/seqid=0 2024-12-10T06:15:39,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742172_1348 (size=30955) 2024-12-10T06:15:39,588 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/61381710d4024806a47185bcfecd51cb 2024-12-10T06:15:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/850de6a8bbb44febb5da0f8e14054ae6 is 50, key is test_row_0/B:col10/1733811339069/Put/seqid=0 2024-12-10T06:15:39,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742173_1349 (size=12001) 2024-12-10T06:15:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T06:15:39,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811399736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811399845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811399865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811399865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:39,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811399875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:40,005 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/850de6a8bbb44febb5da0f8e14054ae6 2024-12-10T06:15:40,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/cbc8b2d1f86c492789029dd488604fdd is 50, key is test_row_0/C:col10/1733811339069/Put/seqid=0 2024-12-10T06:15:40,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742174_1350 (size=12001) 2024-12-10T06:15:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T06:15:40,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811400245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:40,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811400365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:40,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811400385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:40,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811400385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:40,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811400395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:40,424 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/cbc8b2d1f86c492789029dd488604fdd 2024-12-10T06:15:40,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/61381710d4024806a47185bcfecd51cb as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/61381710d4024806a47185bcfecd51cb 2024-12-10T06:15:40,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/61381710d4024806a47185bcfecd51cb, entries=150, sequenceid=36, filesize=30.2 K 2024-12-10T06:15:40,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/850de6a8bbb44febb5da0f8e14054ae6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/850de6a8bbb44febb5da0f8e14054ae6 2024-12-10T06:15:40,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/850de6a8bbb44febb5da0f8e14054ae6, entries=150, sequenceid=36, filesize=11.7 K 2024-12-10T06:15:40,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/cbc8b2d1f86c492789029dd488604fdd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/cbc8b2d1f86c492789029dd488604fdd 2024-12-10T06:15:40,441 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/cbc8b2d1f86c492789029dd488604fdd, entries=150, sequenceid=36, filesize=11.7 K 2024-12-10T06:15:40,442 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1c35e623ec588c4a8cce9c8097008587 in 1290ms, sequenceid=36, compaction requested=false 2024-12-10T06:15:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-10T06:15:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-10T06:15:40,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-10T06:15:40,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4460 sec 2024-12-10T06:15:40,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.4520 sec 2024-12-10T06:15:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T06:15:41,145 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:41,146 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-10T06:15:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T06:15:41,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:41,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:41,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T06:15:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:41,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T06:15:41,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:41,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:41,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:41,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:41,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:41,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:41,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e8a9e5acceff424daa0753c6448f4f35_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811339210/Put/seqid=0 2024-12-10T06:15:41,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742175_1351 (size=14594) 2024-12-10T06:15:41,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T06:15:41,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:41,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811401395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:41,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811401396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:41,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811401405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:41,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811401406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:41,417 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:41,417 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:41,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811401415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:41,420 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:41,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T06:15:41,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:41,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T06:15:41,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811401525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:41,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811401525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:41,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T06:15:41,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:41,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:41,690 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e8a9e5acceff424daa0753c6448f4f35_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e8a9e5acceff424daa0753c6448f4f35_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:41,691 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/4400611a87034b128ff263652340ce43, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:41,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/4400611a87034b128ff263652340ce43 is 175, key is test_row_0/A:col10/1733811339210/Put/seqid=0 2024-12-10T06:15:41,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742176_1352 (size=39549) 2024-12-10T06:15:41,705 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/4400611a87034b128ff263652340ce43 2024-12-10T06:15:41,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/00b22799ae474c29b57d89e7f55d6abd is 50, key is test_row_0/B:col10/1733811339210/Put/seqid=0 2024-12-10T06:15:41,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742177_1353 (size=12001) 2024-12-10T06:15:41,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811401745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:41,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811401745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:41,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T06:15:41,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:41,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T06:15:41,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T06:15:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:41,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:42,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T06:15:42,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:42,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:42,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:42,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811402065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811402065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:42,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/00b22799ae474c29b57d89e7f55d6abd 2024-12-10T06:15:42,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/c97a236b20704ad1a9626727a6fedae3 is 50, key is test_row_0/C:col10/1733811339210/Put/seqid=0 2024-12-10T06:15:42,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742178_1354 (size=12001) 2024-12-10T06:15:42,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/c97a236b20704ad1a9626727a6fedae3 2024-12-10T06:15:42,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/4400611a87034b128ff263652340ce43 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4400611a87034b128ff263652340ce43 2024-12-10T06:15:42,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4400611a87034b128ff263652340ce43, entries=200, sequenceid=51, filesize=38.6 K 2024-12-10T06:15:42,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/00b22799ae474c29b57d89e7f55d6abd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/00b22799ae474c29b57d89e7f55d6abd 2024-12-10T06:15:42,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/00b22799ae474c29b57d89e7f55d6abd, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T06:15:42,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/c97a236b20704ad1a9626727a6fedae3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c97a236b20704ad1a9626727a6fedae3 2024-12-10T06:15:42,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c97a236b20704ad1a9626727a6fedae3, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T06:15:42,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1c35e623ec588c4a8cce9c8097008587 in 869ms, sequenceid=51, compaction requested=true 2024-12-10T06:15:42,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:42,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:42,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:42,141 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:42,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:42,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:42,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:42,141 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:42,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:42,142 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:42,142 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:42,142 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/A is initiating minor compaction (all files) 2024-12-10T06:15:42,142 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/B is initiating minor compaction (all files) 2024-12-10T06:15:42,142 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/A in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:42,142 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/B in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:42,142 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/f7917a425f1942069726be9438b989f5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/850de6a8bbb44febb5da0f8e14054ae6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/00b22799ae474c29b57d89e7f55d6abd] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=35.2 K 2024-12-10T06:15:42,142 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/341f0e0eb6a54c4e93c31e5cc07e038c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/61381710d4024806a47185bcfecd51cb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4400611a87034b128ff263652340ce43] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=99.1 K 2024-12-10T06:15:42,142 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:42,142 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/341f0e0eb6a54c4e93c31e5cc07e038c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/61381710d4024806a47185bcfecd51cb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4400611a87034b128ff263652340ce43] 2024-12-10T06:15:42,143 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 341f0e0eb6a54c4e93c31e5cc07e038c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733811338999 2024-12-10T06:15:42,143 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting f7917a425f1942069726be9438b989f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733811338999 2024-12-10T06:15:42,143 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61381710d4024806a47185bcfecd51cb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733811339033 2024-12-10T06:15:42,143 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 850de6a8bbb44febb5da0f8e14054ae6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733811339033 2024-12-10T06:15:42,144 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b22799ae474c29b57d89e7f55d6abd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811339190 2024-12-10T06:15:42,144 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4400611a87034b128ff263652340ce43, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811339190 2024-12-10T06:15:42,149 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:42,151 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:42,152 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/6ba3317f71974d299dcfdcf95f8359d2 is 50, key is test_row_0/B:col10/1733811339210/Put/seqid=0 2024-12-10T06:15:42,157 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412103f3cb5f884bd4a0f93d87f129547c063_1c35e623ec588c4a8cce9c8097008587 store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:42,159 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412103f3cb5f884bd4a0f93d87f129547c063_1c35e623ec588c4a8cce9c8097008587, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:42,159 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103f3cb5f884bd4a0f93d87f129547c063_1c35e623ec588c4a8cce9c8097008587 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:42,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742179_1355 (size=12104) 2024-12-10T06:15:42,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742180_1356 (size=4469) 2024-12-10T06:15:42,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T06:15:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:42,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T06:15:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210107edd5bb4d84aa7be3e66c722037a50_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811341328/Put/seqid=0 2024-12-10T06:15:42,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742181_1357 (size=12154) 2024-12-10T06:15:42,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T06:15:42,568 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/6ba3317f71974d299dcfdcf95f8359d2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/6ba3317f71974d299dcfdcf95f8359d2 2024-12-10T06:15:42,572 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/B of 1c35e623ec588c4a8cce9c8097008587 into 6ba3317f71974d299dcfdcf95f8359d2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:42,572 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:42,572 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/B, priority=13, startTime=1733811342141; duration=0sec 2024-12-10T06:15:42,572 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:42,572 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:B 2024-12-10T06:15:42,572 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:42,573 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:42,573 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/C is initiating minor compaction (all files) 2024-12-10T06:15:42,574 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/C in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:42,574 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/120d089aba6b46378bfd8e1cc299a9c1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/cbc8b2d1f86c492789029dd488604fdd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c97a236b20704ad1a9626727a6fedae3] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=35.2 K 2024-12-10T06:15:42,574 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#A#compaction#301 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:42,574 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 120d089aba6b46378bfd8e1cc299a9c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733811338999 2024-12-10T06:15:42,575 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc8b2d1f86c492789029dd488604fdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733811339033 2024-12-10T06:15:42,575 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/204bccc796c24ff1b04eeda379db31a3 is 175, key is test_row_0/A:col10/1733811339210/Put/seqid=0 2024-12-10T06:15:42,575 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting c97a236b20704ad1a9626727a6fedae3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811339190 2024-12-10T06:15:42,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742182_1358 (size=31058) 2024-12-10T06:15:42,583 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:42,584 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/53c6100cdb8f48baa39b7687bf403a9a is 50, key is test_row_0/C:col10/1733811339210/Put/seqid=0 2024-12-10T06:15:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:42,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:42,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742183_1359 (size=12104) 2024-12-10T06:15:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:42,640 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210107edd5bb4d84aa7be3e66c722037a50_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210107edd5bb4d84aa7be3e66c722037a50_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:42,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/793565bc14b1486d8f0212621ffd6796, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/793565bc14b1486d8f0212621ffd6796 is 175, key is test_row_0/A:col10/1733811341328/Put/seqid=0 2024-12-10T06:15:42,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742184_1360 (size=30955) 2024-12-10T06:15:42,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811402632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:42,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:42,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811402636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:42,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:42,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811402755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:42,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811402755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:42,985 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/204bccc796c24ff1b04eeda379db31a3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/204bccc796c24ff1b04eeda379db31a3 2024-12-10T06:15:42,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811402975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:42,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811402975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:42,990 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/A of 1c35e623ec588c4a8cce9c8097008587 into 204bccc796c24ff1b04eeda379db31a3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:42,991 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:42,991 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/A, priority=13, startTime=1733811342141; duration=0sec 2024-12-10T06:15:42,991 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:42,991 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:A 2024-12-10T06:15:42,996 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/53c6100cdb8f48baa39b7687bf403a9a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/53c6100cdb8f48baa39b7687bf403a9a 2024-12-10T06:15:43,001 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/C of 1c35e623ec588c4a8cce9c8097008587 into 53c6100cdb8f48baa39b7687bf403a9a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:43,001 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:43,001 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/C, priority=13, startTime=1733811342141; duration=0sec 2024-12-10T06:15:43,001 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:43,001 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:C 2024-12-10T06:15:43,045 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/793565bc14b1486d8f0212621ffd6796 2024-12-10T06:15:43,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/eb39ac186c4f4fea98ad44f4b35ec0dc is 50, key is test_row_0/B:col10/1733811341328/Put/seqid=0 2024-12-10T06:15:43,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742185_1361 (size=12001) 2024-12-10T06:15:43,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/eb39ac186c4f4fea98ad44f4b35ec0dc 2024-12-10T06:15:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/d438da778dfb415588ca4fcdb2269f46 is 50, key is test_row_0/C:col10/1733811341328/Put/seqid=0 2024-12-10T06:15:43,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742186_1362 (size=12001) 2024-12-10T06:15:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T06:15:43,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811403295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:43,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811403295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:43,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811403425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:43,435 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:43,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811403435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811403436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:43,446 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:43,446 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:43,476 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/d438da778dfb415588ca4fcdb2269f46 2024-12-10T06:15:43,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/793565bc14b1486d8f0212621ffd6796 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/793565bc14b1486d8f0212621ffd6796 2024-12-10T06:15:43,483 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/793565bc14b1486d8f0212621ffd6796, entries=150, sequenceid=68, filesize=30.2 K 2024-12-10T06:15:43,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/eb39ac186c4f4fea98ad44f4b35ec0dc as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/eb39ac186c4f4fea98ad44f4b35ec0dc 2024-12-10T06:15:43,490 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/eb39ac186c4f4fea98ad44f4b35ec0dc, entries=150, sequenceid=68, filesize=11.7 K 2024-12-10T06:15:43,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/d438da778dfb415588ca4fcdb2269f46 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d438da778dfb415588ca4fcdb2269f46 2024-12-10T06:15:43,494 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d438da778dfb415588ca4fcdb2269f46, entries=150, sequenceid=68, filesize=11.7 K 2024-12-10T06:15:43,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1c35e623ec588c4a8cce9c8097008587 in 1271ms, sequenceid=68, compaction requested=false 2024-12-10T06:15:43,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:43,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:43,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-10T06:15:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-10T06:15:43,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-10T06:15:43,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3480 sec 2024-12-10T06:15:43,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.3540 sec 2024-12-10T06:15:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:43,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:15:43,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:43,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:43,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:43,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:43,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:43,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:43,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e536e549c6db46ab8c50d0be28f673ea_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811342635/Put/seqid=0 2024-12-10T06:15:43,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742187_1363 (size=14594) 2024-12-10T06:15:43,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811403922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811403936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:44,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811404045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:44,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811404056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:44,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:44,239 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e536e549c6db46ab8c50d0be28f673ea_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e536e549c6db46ab8c50d0be28f673ea_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:44,240 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/4cb98493c6e24bb490cfd11824ad6168, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:44,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/4cb98493c6e24bb490cfd11824ad6168 is 175, key is test_row_0/A:col10/1733811342635/Put/seqid=0 2024-12-10T06:15:44,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742188_1364 (size=39549) 2024-12-10T06:15:44,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811404265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:44,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811404275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:44,442 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:15:44,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811404585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:44,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811404585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:44,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/4cb98493c6e24bb490cfd11824ad6168 2024-12-10T06:15:44,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/8a385fc34d0444e6948e57ef8cc85f43 is 50, key is test_row_0/B:col10/1733811342635/Put/seqid=0 2024-12-10T06:15:44,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742189_1365 (size=12001) 2024-12-10T06:15:45,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/8a385fc34d0444e6948e57ef8cc85f43 2024-12-10T06:15:45,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/7a2e642f0e6f47a49dff50cd911bfb10 is 50, key is test_row_0/C:col10/1733811342635/Put/seqid=0 2024-12-10T06:15:45,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742190_1366 (size=12001) 2024-12-10T06:15:45,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811405105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:45,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811405105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T06:15:45,320 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:45,332 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-10T06:15:45,335 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T06:15:45,336 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:45,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T06:15:45,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/7a2e642f0e6f47a49dff50cd911bfb10 2024-12-10T06:15:45,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/4cb98493c6e24bb490cfd11824ad6168 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4cb98493c6e24bb490cfd11824ad6168 2024-12-10T06:15:45,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T06:15:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:45,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:45,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4cb98493c6e24bb490cfd11824ad6168, entries=200, sequenceid=87, filesize=38.6 K 2024-12-10T06:15:45,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/8a385fc34d0444e6948e57ef8cc85f43 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/8a385fc34d0444e6948e57ef8cc85f43 2024-12-10T06:15:45,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/8a385fc34d0444e6948e57ef8cc85f43, entries=150, sequenceid=87, filesize=11.7 K 2024-12-10T06:15:45,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/7a2e642f0e6f47a49dff50cd911bfb10 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7a2e642f0e6f47a49dff50cd911bfb10 2024-12-10T06:15:45,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7a2e642f0e6f47a49dff50cd911bfb10, entries=150, sequenceid=87, filesize=11.7 K 2024-12-10T06:15:45,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1c35e623ec588c4a8cce9c8097008587 in 1686ms, sequenceid=87, compaction requested=true 2024-12-10T06:15:45,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:45,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:45,510 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:45,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:45,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:45,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:45,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:45,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T06:15:45,510 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:45,516 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:45,516 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:45,516 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/A is initiating minor compaction (all files) 2024-12-10T06:15:45,516 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/B is initiating minor compaction (all files) 2024-12-10T06:15:45,516 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/A in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:45,516 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/B in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:45,516 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/204bccc796c24ff1b04eeda379db31a3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/793565bc14b1486d8f0212621ffd6796, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4cb98493c6e24bb490cfd11824ad6168] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=99.2 K 2024-12-10T06:15:45,516 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/6ba3317f71974d299dcfdcf95f8359d2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/eb39ac186c4f4fea98ad44f4b35ec0dc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/8a385fc34d0444e6948e57ef8cc85f43] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=35.3 K 2024-12-10T06:15:45,516 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:45,516 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/204bccc796c24ff1b04eeda379db31a3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/793565bc14b1486d8f0212621ffd6796, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4cb98493c6e24bb490cfd11824ad6168] 2024-12-10T06:15:45,517 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba3317f71974d299dcfdcf95f8359d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811339190 2024-12-10T06:15:45,517 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 204bccc796c24ff1b04eeda379db31a3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811339190 2024-12-10T06:15:45,517 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 793565bc14b1486d8f0212621ffd6796, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733811341328 2024-12-10T06:15:45,517 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting eb39ac186c4f4fea98ad44f4b35ec0dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733811341328 2024-12-10T06:15:45,518 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cb98493c6e24bb490cfd11824ad6168, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733811342603 2024-12-10T06:15:45,518 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a385fc34d0444e6948e57ef8cc85f43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733811342620 2024-12-10T06:15:45,526 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:45,528 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:45,529 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/e363efe38abc43fdbce46f76c2da93b4 is 50, key is test_row_0/B:col10/1733811342635/Put/seqid=0 2024-12-10T06:15:45,531 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210cb1040dc5d6b4ca1b97c9ef7e1c137fb_1c35e623ec588c4a8cce9c8097008587 store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:45,533 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210cb1040dc5d6b4ca1b97c9ef7e1c137fb_1c35e623ec588c4a8cce9c8097008587, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:45,533 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cb1040dc5d6b4ca1b97c9ef7e1c137fb_1c35e623ec588c4a8cce9c8097008587 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:45,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742191_1367 (size=12207) 2024-12-10T06:15:45,548 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/e363efe38abc43fdbce46f76c2da93b4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/e363efe38abc43fdbce46f76c2da93b4 2024-12-10T06:15:45,553 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/B of 1c35e623ec588c4a8cce9c8097008587 into e363efe38abc43fdbce46f76c2da93b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:45,553 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:45,554 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/B, priority=13, startTime=1733811345510; duration=0sec 2024-12-10T06:15:45,554 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:45,554 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:B 2024-12-10T06:15:45,554 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:45,557 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:45,557 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/C is initiating minor compaction (all files) 2024-12-10T06:15:45,557 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/C in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:45,557 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/53c6100cdb8f48baa39b7687bf403a9a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d438da778dfb415588ca4fcdb2269f46, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7a2e642f0e6f47a49dff50cd911bfb10] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=35.3 K 2024-12-10T06:15:45,558 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c6100cdb8f48baa39b7687bf403a9a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733811339190 2024-12-10T06:15:45,558 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d438da778dfb415588ca4fcdb2269f46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733811341328 2024-12-10T06:15:45,559 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a2e642f0e6f47a49dff50cd911bfb10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733811342620 2024-12-10T06:15:45,568 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:45,569 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/2ef8c2930735431da280511d3fce05c7 is 50, key is test_row_0/C:col10/1733811342635/Put/seqid=0 2024-12-10T06:15:45,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742192_1368 (size=4469) 2024-12-10T06:15:45,574 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#A#compaction#310 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:45,575 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b8d7d6bc33ff4a22abe6fc689da10011 is 175, key is test_row_0/A:col10/1733811342635/Put/seqid=0 2024-12-10T06:15:45,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742194_1370 (size=31161) 2024-12-10T06:15:45,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742193_1369 (size=12207) 2024-12-10T06:15:45,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T06:15:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:45,644 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:15:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:45,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210660d7831be68473a98e377e3b3bca0a1_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811343919/Put/seqid=0 2024-12-10T06:15:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T06:15:45,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742195_1371 (size=12154) 2024-12-10T06:15:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:45,680 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210660d7831be68473a98e377e3b3bca0a1_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210660d7831be68473a98e377e3b3bca0a1_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:45,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/0b221344c2a24a5d990b72b4254e6379, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:45,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/0b221344c2a24a5d990b72b4254e6379 is 175, key is test_row_0/A:col10/1733811343919/Put/seqid=0 2024-12-10T06:15:45,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742196_1372 (size=30955) 2024-12-10T06:15:45,686 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/0b221344c2a24a5d990b72b4254e6379 2024-12-10T06:15:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/b44dcce70ca9495289f566a4f5c3f349 is 50, key is test_row_0/B:col10/1733811343919/Put/seqid=0 2024-12-10T06:15:45,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742197_1373 (size=12001) 2024-12-10T06:15:45,707 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/b44dcce70ca9495289f566a4f5c3f349 2024-12-10T06:15:45,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/0ec6bc8c5c1d4728a8399a218f4aa2d3 is 50, key is test_row_0/C:col10/1733811343919/Put/seqid=0 2024-12-10T06:15:45,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742198_1374 (size=12001) 2024-12-10T06:15:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T06:15:45,995 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b8d7d6bc33ff4a22abe6fc689da10011 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b8d7d6bc33ff4a22abe6fc689da10011 2024-12-10T06:15:45,998 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/2ef8c2930735431da280511d3fce05c7 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/2ef8c2930735431da280511d3fce05c7 2024-12-10T06:15:45,999 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/A of 1c35e623ec588c4a8cce9c8097008587 into b8d7d6bc33ff4a22abe6fc689da10011(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:45,999 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:45,999 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/A, priority=13, startTime=1733811345510; duration=0sec 2024-12-10T06:15:45,999 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:46,000 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:A 2024-12-10T06:15:46,002 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/C of 1c35e623ec588c4a8cce9c8097008587 into 2ef8c2930735431da280511d3fce05c7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:46,002 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:46,002 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/C, priority=13, startTime=1733811345510; duration=0sec 2024-12-10T06:15:46,003 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:46,003 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:C 2024-12-10T06:15:46,124 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/0ec6bc8c5c1d4728a8399a218f4aa2d3 2024-12-10T06:15:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:46,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:46,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/0b221344c2a24a5d990b72b4254e6379 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/0b221344c2a24a5d990b72b4254e6379 2024-12-10T06:15:46,147 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/0b221344c2a24a5d990b72b4254e6379, entries=150, sequenceid=104, filesize=30.2 K 2024-12-10T06:15:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/b44dcce70ca9495289f566a4f5c3f349 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b44dcce70ca9495289f566a4f5c3f349 2024-12-10T06:15:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,156 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b44dcce70ca9495289f566a4f5c3f349, entries=150, sequenceid=104, filesize=11.7 K 2024-12-10T06:15:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/0ec6bc8c5c1d4728a8399a218f4aa2d3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/0ec6bc8c5c1d4728a8399a218f4aa2d3 2024-12-10T06:15:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,169 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/0ec6bc8c5c1d4728a8399a218f4aa2d3, entries=150, sequenceid=104, filesize=11.7 K 2024-12-10T06:15:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,170 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=33.54 KB/34350 for 1c35e623ec588c4a8cce9c8097008587 in 526ms, sequenceid=104, compaction requested=false 2024-12-10T06:15:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:46,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-10T06:15:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-10T06:15:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-10T06:15:46,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 836 msec 2024-12-10T06:15:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 844 msec 2024-12-10T06:15:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:46,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:15:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210247673981a9342ee95de35508694688a_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811346172/Put/seqid=0 2024-12-10T06:15:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742199_1375 (size=19474) 2024-12-10T06:15:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,208 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,213 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210247673981a9342ee95de35508694688a_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210247673981a9342ee95de35508694688a_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:46,214 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/65d78fdcccca4793b435b650445fb265, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:46,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/65d78fdcccca4793b435b650445fb265 is 175, key is test_row_0/A:col10/1733811346172/Put/seqid=0 2024-12-10T06:15:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742200_1376 (size=56733) 2024-12-10T06:15:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,225 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/65d78fdcccca4793b435b650445fb265 2024-12-10T06:15:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/80edb539968e4b5a97ce11c37401ed47 is 50, key is test_row_0/B:col10/1733811346172/Put/seqid=0 2024-12-10T06:15:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742201_1377 (size=19021) 2024-12-10T06:15:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:46,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811346243, max timestamp=1733811346310Num uniques 859; 2024-12-10T06:15:46,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811346243, max timestamp=1733811346310Num uniques 859; 2024-12-10T06:15:46,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811346243, max timestamp=1733811346310Num uniques 859; 2024-12-10T06:15:46,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:46,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:46,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:46,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811406419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:46,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811406425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T06:15:46,475 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:46,476 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-10T06:15:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T06:15:46,477 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:46,478 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:46,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:46,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811406545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:46,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811406546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:46,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:46,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T06:15:46,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T06:15:46,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:46,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:46,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:46,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:46,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:46,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/80edb539968e4b5a97ce11c37401ed47 2024-12-10T06:15:46,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/c8de26173d1241fe9d201b9a863d9463 is 50, key is test_row_0/C:col10/1733811346172/Put/seqid=0 2024-12-10T06:15:46,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742202_1378 (size=12001) 2024-12-10T06:15:46,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811406765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:46,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811406765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:46,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T06:15:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:46,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T06:15:46,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T06:15:46,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:46,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:46,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:46,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:46,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:47,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/c8de26173d1241fe9d201b9a863d9463 2024-12-10T06:15:47,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/65d78fdcccca4793b435b650445fb265 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/65d78fdcccca4793b435b650445fb265 2024-12-10T06:15:47,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/65d78fdcccca4793b435b650445fb265, entries=300, sequenceid=116, filesize=55.4 K 2024-12-10T06:15:47,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/80edb539968e4b5a97ce11c37401ed47 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/80edb539968e4b5a97ce11c37401ed47 2024-12-10T06:15:47,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/80edb539968e4b5a97ce11c37401ed47, entries=300, sequenceid=116, filesize=18.6 K 2024-12-10T06:15:47,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/c8de26173d1241fe9d201b9a863d9463 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c8de26173d1241fe9d201b9a863d9463 2024-12-10T06:15:47,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c8de26173d1241fe9d201b9a863d9463, entries=150, sequenceid=116, filesize=11.7 K 2024-12-10T06:15:47,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1c35e623ec588c4a8cce9c8097008587 in 892ms, sequenceid=116, compaction requested=true 2024-12-10T06:15:47,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:47,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:47,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:47,078 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:47,078 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:47,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:47,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:47,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:47,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:47,080 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118849 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:47,080 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:47,081 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/B is initiating minor compaction (all files) 2024-12-10T06:15:47,081 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/A is initiating minor compaction (all files) 2024-12-10T06:15:47,081 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/B in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:47,081 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/A in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:47,081 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/e363efe38abc43fdbce46f76c2da93b4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b44dcce70ca9495289f566a4f5c3f349, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/80edb539968e4b5a97ce11c37401ed47] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=42.2 K 2024-12-10T06:15:47,081 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b8d7d6bc33ff4a22abe6fc689da10011, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/0b221344c2a24a5d990b72b4254e6379, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/65d78fdcccca4793b435b650445fb265] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=116.1 K 2024-12-10T06:15:47,081 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:47,081 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b8d7d6bc33ff4a22abe6fc689da10011, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/0b221344c2a24a5d990b72b4254e6379, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/65d78fdcccca4793b435b650445fb265] 2024-12-10T06:15:47,081 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting e363efe38abc43fdbce46f76c2da93b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733811342620 2024-12-10T06:15:47,081 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8d7d6bc33ff4a22abe6fc689da10011, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733811342620 2024-12-10T06:15:47,082 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b221344c2a24a5d990b72b4254e6379, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733811343919 2024-12-10T06:15:47,082 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b44dcce70ca9495289f566a4f5c3f349, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733811343919 2024-12-10T06:15:47,082 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65d78fdcccca4793b435b650445fb265, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733811346142 2024-12-10T06:15:47,082 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 80edb539968e4b5a97ce11c37401ed47, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733811346142 2024-12-10T06:15:47,089 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:47,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T06:15:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:47,090 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-10T06:15:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:47,091 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#B#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:47,091 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/abb3760d694149eba037a26d34ac8c7b is 50, key is test_row_0/B:col10/1733811346172/Put/seqid=0 2024-12-10T06:15:47,093 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210bec2f63015414c16baa760f4c938778e_1c35e623ec588c4a8cce9c8097008587 store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:47,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742203_1379 (size=12309) 2024-12-10T06:15:47,095 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210bec2f63015414c16baa760f4c938778e_1c35e623ec588c4a8cce9c8097008587, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:47,096 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bec2f63015414c16baa760f4c938778e_1c35e623ec588c4a8cce9c8097008587 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d4bd70d1fc9b45c496691d5b71f43c41_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811346381/Put/seqid=0 2024-12-10T06:15:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T06:15:47,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742204_1380 (size=4469) 2024-12-10T06:15:47,109 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/abb3760d694149eba037a26d34ac8c7b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/abb3760d694149eba037a26d34ac8c7b 2024-12-10T06:15:47,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742205_1381 (size=14794) 2024-12-10T06:15:47,113 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/B of 1c35e623ec588c4a8cce9c8097008587 into abb3760d694149eba037a26d34ac8c7b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:47,113 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:47,113 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/B, priority=13, startTime=1733811347078; duration=0sec 2024-12-10T06:15:47,113 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:47,113 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:B 2024-12-10T06:15:47,113 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:47,115 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:47,115 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/C is initiating minor compaction (all files) 2024-12-10T06:15:47,115 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/C in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:47,115 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/2ef8c2930735431da280511d3fce05c7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/0ec6bc8c5c1d4728a8399a218f4aa2d3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c8de26173d1241fe9d201b9a863d9463] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=35.4 K 2024-12-10T06:15:47,115 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef8c2930735431da280511d3fce05c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733811342620 2024-12-10T06:15:47,116 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ec6bc8c5c1d4728a8399a218f4aa2d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733811343919 2024-12-10T06:15:47,116 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting c8de26173d1241fe9d201b9a863d9463, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733811346144 2024-12-10T06:15:47,123 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#C#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:47,123 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/7236fa9b2b8d43e5847b93b3bb7f98e8 is 50, key is test_row_0/C:col10/1733811346172/Put/seqid=0 2024-12-10T06:15:47,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742206_1382 (size=12309) 2024-12-10T06:15:47,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811407120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811407134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811407236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811407245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811407455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811407456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811407475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811407475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811407475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,492 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,492 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,493 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,510 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:47,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:47,511 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/87c9a76d48db47319567239155585edb is 175, key is test_row_0/A:col10/1733811346172/Put/seqid=0 2024-12-10T06:15:47,514 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d4bd70d1fc9b45c496691d5b71f43c41_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d4bd70d1fc9b45c496691d5b71f43c41_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:47,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742207_1383 (size=31263) 2024-12-10T06:15:47,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/f435a5fd179441da9dd3ff183620b8c4, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:47,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/f435a5fd179441da9dd3ff183620b8c4 is 175, key is test_row_0/A:col10/1733811346381/Put/seqid=0 2024-12-10T06:15:47,519 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/87c9a76d48db47319567239155585edb as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/87c9a76d48db47319567239155585edb 2024-12-10T06:15:47,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742208_1384 (size=39749) 2024-12-10T06:15:47,524 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/A of 1c35e623ec588c4a8cce9c8097008587 into 87c9a76d48db47319567239155585edb(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:47,524 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:47,524 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/A, priority=13, startTime=1733811347078; duration=0sec 2024-12-10T06:15:47,524 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:47,524 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:A 2024-12-10T06:15:47,528 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/f435a5fd179441da9dd3ff183620b8c4 2024-12-10T06:15:47,531 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/7236fa9b2b8d43e5847b93b3bb7f98e8 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7236fa9b2b8d43e5847b93b3bb7f98e8 2024-12-10T06:15:47,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d4b63d9cc4fb446db36aba2c0081d26e is 50, key is test_row_0/B:col10/1733811346381/Put/seqid=0 2024-12-10T06:15:47,541 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/C of 1c35e623ec588c4a8cce9c8097008587 into 7236fa9b2b8d43e5847b93b3bb7f98e8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:47,541 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:47,541 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/C, priority=13, startTime=1733811347078; duration=0sec 2024-12-10T06:15:47,541 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:47,541 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:C 2024-12-10T06:15:47,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742209_1385 (size=12151) 2024-12-10T06:15:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T06:15:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811407775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811407775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:47,943 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d4b63d9cc4fb446db36aba2c0081d26e 2024-12-10T06:15:47,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/ca33f96bab4a479bbc13905e7cbabb7d is 50, key is test_row_0/C:col10/1733811346381/Put/seqid=0 2024-12-10T06:15:47,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742210_1386 (size=12151) 2024-12-10T06:15:48,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811408295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:48,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811408295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:48,358 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/ca33f96bab4a479bbc13905e7cbabb7d 2024-12-10T06:15:48,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/f435a5fd179441da9dd3ff183620b8c4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/f435a5fd179441da9dd3ff183620b8c4 2024-12-10T06:15:48,366 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/f435a5fd179441da9dd3ff183620b8c4, entries=200, sequenceid=153, filesize=38.8 K 2024-12-10T06:15:48,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d4b63d9cc4fb446db36aba2c0081d26e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d4b63d9cc4fb446db36aba2c0081d26e 2024-12-10T06:15:48,369 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d4b63d9cc4fb446db36aba2c0081d26e, entries=150, sequenceid=153, filesize=11.9 K 2024-12-10T06:15:48,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/ca33f96bab4a479bbc13905e7cbabb7d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/ca33f96bab4a479bbc13905e7cbabb7d 2024-12-10T06:15:48,373 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/ca33f96bab4a479bbc13905e7cbabb7d, entries=150, sequenceid=153, filesize=11.9 K 2024-12-10T06:15:48,374 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 1c35e623ec588c4a8cce9c8097008587 in 1284ms, sequenceid=153, compaction requested=false 2024-12-10T06:15:48,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:48,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:48,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-10T06:15:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-10T06:15:48,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-10T06:15:48,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-12-10T06:15:48,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.9020 sec 2024-12-10T06:15:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T06:15:48,625 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:48,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-10T06:15:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T06:15:48,627 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:48,628 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:48,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T06:15:48,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T06:15:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:48,781 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T06:15:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:48,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105d04c0cdcd754b459e54dbd57ec98c0a_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811347132/Put/seqid=0 2024-12-10T06:15:48,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742211_1387 (size=12304) 2024-12-10T06:15:48,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:48,801 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105d04c0cdcd754b459e54dbd57ec98c0a_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105d04c0cdcd754b459e54dbd57ec98c0a_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:48,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/ee467c557c164e25b76c7eff850fa5d5, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/ee467c557c164e25b76c7eff850fa5d5 is 175, key is test_row_0/A:col10/1733811347132/Put/seqid=0 2024-12-10T06:15:48,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742212_1388 (size=31105) 2024-12-10T06:15:48,809 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/ee467c557c164e25b76c7eff850fa5d5 2024-12-10T06:15:48,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/1d01b4dd0963497c82c6e81ab7c3f686 is 50, key is test_row_0/B:col10/1733811347132/Put/seqid=0 2024-12-10T06:15:48,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742213_1389 (size=12151) 2024-12-10T06:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T06:15:49,222 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/1d01b4dd0963497c82c6e81ab7c3f686 2024-12-10T06:15:49,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/d01d28ea48024e82b437dbaad2cba20b is 50, key is test_row_0/C:col10/1733811347132/Put/seqid=0 2024-12-10T06:15:49,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742214_1390 (size=12151) 2024-12-10T06:15:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T06:15:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:49,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811349391, max timestamp=1733811349412Num uniques 859; 2024-12-10T06:15:49,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:49,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811349391, max timestamp=1733811349412Num uniques 859; 2024-12-10T06:15:49,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811349391, max timestamp=1733811349412Num uniques 859; 2024-12-10T06:15:49,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:49,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:49,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811409506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:49,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811409507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:49,633 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/d01d28ea48024e82b437dbaad2cba20b 2024-12-10T06:15:49,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/ee467c557c164e25b76c7eff850fa5d5 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ee467c557c164e25b76c7eff850fa5d5 2024-12-10T06:15:49,641 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ee467c557c164e25b76c7eff850fa5d5, entries=150, sequenceid=165, filesize=30.4 K 2024-12-10T06:15:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/1d01b4dd0963497c82c6e81ab7c3f686 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1d01b4dd0963497c82c6e81ab7c3f686 2024-12-10T06:15:49,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811409625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:49,645 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1d01b4dd0963497c82c6e81ab7c3f686, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T06:15:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/d01d28ea48024e82b437dbaad2cba20b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d01d28ea48024e82b437dbaad2cba20b 2024-12-10T06:15:49,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811409635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:49,649 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d01d28ea48024e82b437dbaad2cba20b, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T06:15:49,649 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 1c35e623ec588c4a8cce9c8097008587 in 868ms, sequenceid=165, compaction requested=true 2024-12-10T06:15:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-10T06:15:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-10T06:15:49,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-10T06:15:49,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0220 sec 2024-12-10T06:15:49,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.0260 sec 2024-12-10T06:15:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T06:15:49,765 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:49,765 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-10T06:15:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T06:15:49,767 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:49,768 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:49,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:49,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-10T06:15:49,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:49,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T06:15:49,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:49,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T06:15:49,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:49,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T06:15:49,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a56643bf064b46f194444a20f3dee414_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811349855/Put/seqid=0 2024-12-10T06:15:49,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742215_1391 (size=14794) 2024-12-10T06:15:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T06:15:49,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811409884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811409898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:49,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:49,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:49,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811410015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:50,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811410025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:50,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T06:15:50,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811410235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811410245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:50,270 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:50,274 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a56643bf064b46f194444a20f3dee414_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a56643bf064b46f194444a20f3dee414_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:50,274 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/ec01e38e97b1406ba41cbe1ea963d827, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:50,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/ec01e38e97b1406ba41cbe1ea963d827 is 175, key is test_row_0/A:col10/1733811349855/Put/seqid=0 2024-12-10T06:15:50,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742216_1392 (size=39749) 2024-12-10T06:15:50,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:50,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:50,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T06:15:50,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:50,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:50,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:50,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811410555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:50,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:50,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811410565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:50,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:50,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:50,679 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/ec01e38e97b1406ba41cbe1ea963d827 2024-12-10T06:15:50,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:50,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:50,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/7e98ce8dbd994ac48e5480976d3176e6 is 50, key is test_row_0/B:col10/1733811349855/Put/seqid=0 2024-12-10T06:15:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742217_1393 (size=12151) 2024-12-10T06:15:50,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T06:15:50,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:50,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811411065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:51,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/7e98ce8dbd994ac48e5480976d3176e6 2024-12-10T06:15:51,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/11b006ebded8400dadb9dea3a53d9624 is 50, key is test_row_0/C:col10/1733811349855/Put/seqid=0 2024-12-10T06:15:51,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811411085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:51,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:51,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:51,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:51,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742218_1394 (size=12151) 2024-12-10T06:15:51,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:51,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/11b006ebded8400dadb9dea3a53d9624 2024-12-10T06:15:51,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/ec01e38e97b1406ba41cbe1ea963d827 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ec01e38e97b1406ba41cbe1ea963d827 2024-12-10T06:15:51,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ec01e38e97b1406ba41cbe1ea963d827, entries=200, sequenceid=203, filesize=38.8 K 2024-12-10T06:15:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/7e98ce8dbd994ac48e5480976d3176e6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/7e98ce8dbd994ac48e5480976d3176e6 2024-12-10T06:15:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/7e98ce8dbd994ac48e5480976d3176e6, entries=150, sequenceid=203, filesize=11.9 K 2024-12-10T06:15:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/11b006ebded8400dadb9dea3a53d9624 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/11b006ebded8400dadb9dea3a53d9624 2024-12-10T06:15:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/11b006ebded8400dadb9dea3a53d9624, entries=150, sequenceid=203, filesize=11.9 K 2024-12-10T06:15:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 1c35e623ec588c4a8cce9c8097008587 in 1664ms, sequenceid=203, compaction requested=true 2024-12-10T06:15:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,521 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:51,522 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,523 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141866 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:51,523 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/A is initiating minor compaction (all files) 2024-12-10T06:15:51,523 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/A in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,523 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/87c9a76d48db47319567239155585edb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/f435a5fd179441da9dd3ff183620b8c4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ee467c557c164e25b76c7eff850fa5d5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ec01e38e97b1406ba41cbe1ea963d827] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=138.5 K 2024-12-10T06:15:51,523 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,523 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/87c9a76d48db47319567239155585edb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/f435a5fd179441da9dd3ff183620b8c4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ee467c557c164e25b76c7eff850fa5d5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ec01e38e97b1406ba41cbe1ea963d827] 2024-12-10T06:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,523 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,523 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/B is initiating minor compaction (all files) 2024-12-10T06:15:51,524 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c9a76d48db47319567239155585edb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733811346144 2024-12-10T06:15:51,524 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/B in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,524 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/abb3760d694149eba037a26d34ac8c7b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d4b63d9cc4fb446db36aba2c0081d26e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1d01b4dd0963497c82c6e81ab7c3f686, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/7e98ce8dbd994ac48e5480976d3176e6] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=47.6 K 2024-12-10T06:15:51,524 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f435a5fd179441da9dd3ff183620b8c4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733811346381 2024-12-10T06:15:51,524 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting abb3760d694149eba037a26d34ac8c7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733811346144 2024-12-10T06:15:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,524 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b63d9cc4fb446db36aba2c0081d26e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733811346381 2024-12-10T06:15:51,524 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee467c557c164e25b76c7eff850fa5d5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733811347114 2024-12-10T06:15:51,525 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d01b4dd0963497c82c6e81ab7c3f686, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733811347114 2024-12-10T06:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,525 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec01e38e97b1406ba41cbe1ea963d827, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733811349482 2024-12-10T06:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,525 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e98ce8dbd994ac48e5480976d3176e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733811349484 2024-12-10T06:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,533 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,536 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,536 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/657f7faa05334ab2a7e4764ba03d4805 is 50, key is test_row_0/B:col10/1733811349855/Put/seqid=0 2024-12-10T06:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,537 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412103b0614298e1a4ace9d3416c4ba5e9162_1c35e623ec588c4a8cce9c8097008587 store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,540 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412103b0614298e1a4ace9d3416c4ba5e9162_1c35e623ec588c4a8cce9c8097008587, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,540 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103b0614298e1a4ace9d3416c4ba5e9162_1c35e623ec588c4a8cce9c8097008587 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742219_1395 (size=12595) 2024-12-10T06:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742220_1396 (size=4469) 2024-12-10T06:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T06:15:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,606 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T06:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103e10af84e6d84180b2b225870febd79c_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811349883/Put/seqid=0 2024-12-10T06:15:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742221_1397 (size=9814) 2024-12-10T06:15:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T06:15:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,945 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,947 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/eb6341c79a9340d9963e3a72f4bb08f9 is 175, key is test_row_0/A:col10/1733811349855/Put/seqid=0 2024-12-10T06:15:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,949 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/657f7faa05334ab2a7e4764ba03d4805 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/657f7faa05334ab2a7e4764ba03d4805 2024-12-10T06:15:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742222_1398 (size=31549) 2024-12-10T06:15:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,955 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/B of 1c35e623ec588c4a8cce9c8097008587 into 657f7faa05334ab2a7e4764ba03d4805(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:51,955 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:51,955 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/B, priority=12, startTime=1733811351522; duration=0sec 2024-12-10T06:15:51,955 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:51,955 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:B 2024-12-10T06:15:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,955 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,957 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,957 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/C is initiating minor compaction (all files) 2024-12-10T06:15:51,957 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/C in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:51,957 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7236fa9b2b8d43e5847b93b3bb7f98e8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/ca33f96bab4a479bbc13905e7cbabb7d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d01d28ea48024e82b437dbaad2cba20b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/11b006ebded8400dadb9dea3a53d9624] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=47.6 K 2024-12-10T06:15:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,958 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7236fa9b2b8d43e5847b93b3bb7f98e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733811346144 2024-12-10T06:15:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,959 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ca33f96bab4a479bbc13905e7cbabb7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733811346381 2024-12-10T06:15:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,962 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d01d28ea48024e82b437dbaad2cba20b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733811347114 2024-12-10T06:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,963 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b006ebded8400dadb9dea3a53d9624, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733811349484 2024-12-10T06:15:51,963 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/eb6341c79a9340d9963e3a72f4bb08f9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/eb6341c79a9340d9963e3a72f4bb08f9 2024-12-10T06:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,969 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/A of 1c35e623ec588c4a8cce9c8097008587 into eb6341c79a9340d9963e3a72f4bb08f9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:51,969 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:51,969 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/A, priority=12, startTime=1733811351521; duration=0sec 2024-12-10T06:15:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,969 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:51,969 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:A 2024-12-10T06:15:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,974 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,975 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/09e9d7ea2f9a4062a65acc9c63c12805 is 50, key is test_row_0/C:col10/1733811349855/Put/seqid=0 2024-12-10T06:15:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742223_1399 (size=12595) 2024-12-10T06:15:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,011 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/09e9d7ea2f9a4062a65acc9c63c12805 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/09e9d7ea2f9a4062a65acc9c63c12805 2024-12-10T06:15:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,016 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/C of 1c35e623ec588c4a8cce9c8097008587 into 09e9d7ea2f9a4062a65acc9c63c12805(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:52,016 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:52,016 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/C, priority=12, startTime=1733811351522; duration=0sec 2024-12-10T06:15:52,016 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:52,016 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:C 2024-12-10T06:15:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,037 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103e10af84e6d84180b2b225870febd79c_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103e10af84e6d84180b2b225870febd79c_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/edaaf69ad8574c8384190685561b0f57, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/edaaf69ad8574c8384190685561b0f57 is 175, key is test_row_0/A:col10/1733811349883/Put/seqid=0 2024-12-10T06:15:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742224_1400 (size=22461) 2024-12-10T06:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,046 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/edaaf69ad8574c8384190685561b0f57 2024-12-10T06:15:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/ae64938d619d4ba8a19cdf2c9bc59ae8 is 50, key is test_row_0/B:col10/1733811349883/Put/seqid=0 2024-12-10T06:15:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742225_1401 (size=9757) 2024-12-10T06:15:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/ae64938d619d4ba8a19cdf2c9bc59ae8 2024-12-10T06:15:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/9e8a04ad8ed94970918cc5c004ffad62 is 50, key is test_row_0/C:col10/1733811349883/Put/seqid=0 2024-12-10T06:15:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742226_1402 (size=9757) 2024-12-10T06:15:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/9e8a04ad8ed94970918cc5c004ffad62 2024-12-10T06:15:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/edaaf69ad8574c8384190685561b0f57 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/edaaf69ad8574c8384190685561b0f57 2024-12-10T06:15:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,093 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/edaaf69ad8574c8384190685561b0f57, entries=100, sequenceid=211, filesize=21.9 K 2024-12-10T06:15:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/ae64938d619d4ba8a19cdf2c9bc59ae8 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/ae64938d619d4ba8a19cdf2c9bc59ae8 2024-12-10T06:15:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,098 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/ae64938d619d4ba8a19cdf2c9bc59ae8, entries=100, sequenceid=211, filesize=9.5 K 2024-12-10T06:15:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/9e8a04ad8ed94970918cc5c004ffad62 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/9e8a04ad8ed94970918cc5c004ffad62 2024-12-10T06:15:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/9e8a04ad8ed94970918cc5c004ffad62, entries=100, sequenceid=211, filesize=9.5 K 2024-12-10T06:15:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,104 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1c35e623ec588c4a8cce9c8097008587 in 498ms, sequenceid=211, compaction requested=false 2024-12-10T06:15:52,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:52,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-10T06:15:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-10T06:15:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-10T06:15:52,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3370 sec 2024-12-10T06:15:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3430 sec 2024-12-10T06:15:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:52,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100dc717e0be9047899ea3686a79f8b934_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811352270/Put/seqid=0 2024-12-10T06:15:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742227_1403 (size=17284) 2024-12-10T06:15:52,302 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:52,308 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100dc717e0be9047899ea3686a79f8b934_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100dc717e0be9047899ea3686a79f8b934_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:52,309 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b79bf70140c5466dbb8032a4832cc82d, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:52,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b79bf70140c5466dbb8032a4832cc82d is 175, key is test_row_0/A:col10/1733811352270/Put/seqid=0 2024-12-10T06:15:52,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742228_1404 (size=48389) 2024-12-10T06:15:52,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b79bf70140c5466dbb8032a4832cc82d 2024-12-10T06:15:52,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/2978ab31d5534dacbde605ccfc149361 is 50, key is test_row_0/B:col10/1733811352270/Put/seqid=0 2024-12-10T06:15:52,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742229_1405 (size=12151) 2024-12-10T06:15:52,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811352343, max timestamp=1733811352395Num uniques 859; 2024-12-10T06:15:52,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:52,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811352343, max timestamp=1733811352395Num uniques 859; 2024-12-10T06:15:52,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:52,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811352343, max timestamp=1733811352395Num uniques 859; 2024-12-10T06:15:52,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:52,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811412484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:52,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811412489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:52,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811412595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811412615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:52,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/2978ab31d5534dacbde605ccfc149361 2024-12-10T06:15:52,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/a1d4bb645e8c441799d387d209d4921c is 50, key is test_row_0/C:col10/1733811352270/Put/seqid=0 2024-12-10T06:15:52,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742230_1406 (size=12151) 2024-12-10T06:15:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811412815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:52,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811412835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:53,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811413135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:53,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811413155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:53,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/a1d4bb645e8c441799d387d209d4921c 2024-12-10T06:15:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b79bf70140c5466dbb8032a4832cc82d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b79bf70140c5466dbb8032a4832cc82d 2024-12-10T06:15:53,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b79bf70140c5466dbb8032a4832cc82d, entries=250, sequenceid=224, filesize=47.3 K 2024-12-10T06:15:53,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/2978ab31d5534dacbde605ccfc149361 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/2978ab31d5534dacbde605ccfc149361 2024-12-10T06:15:53,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/2978ab31d5534dacbde605ccfc149361, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T06:15:53,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/a1d4bb645e8c441799d387d209d4921c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a1d4bb645e8c441799d387d209d4921c 2024-12-10T06:15:53,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a1d4bb645e8c441799d387d209d4921c, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T06:15:53,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1c35e623ec588c4a8cce9c8097008587 in 920ms, sequenceid=224, compaction requested=true 2024-12-10T06:15:53,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:53,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:53,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:53,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:53,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:53,192 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:53,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:53,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:53,192 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:53,193 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:53,193 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/A is initiating minor compaction (all files) 2024-12-10T06:15:53,193 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/A in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:53,193 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/eb6341c79a9340d9963e3a72f4bb08f9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/edaaf69ad8574c8384190685561b0f57, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b79bf70140c5466dbb8032a4832cc82d] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=100.0 K 2024-12-10T06:15:53,193 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:53,193 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/eb6341c79a9340d9963e3a72f4bb08f9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/edaaf69ad8574c8384190685561b0f57, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b79bf70140c5466dbb8032a4832cc82d] 2024-12-10T06:15:53,194 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:53,194 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/B is initiating minor compaction (all files) 2024-12-10T06:15:53,194 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/B in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:53,194 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/657f7faa05334ab2a7e4764ba03d4805, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/ae64938d619d4ba8a19cdf2c9bc59ae8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/2978ab31d5534dacbde605ccfc149361] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=33.7 K 2024-12-10T06:15:53,194 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb6341c79a9340d9963e3a72f4bb08f9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733811349484 2024-12-10T06:15:53,194 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 657f7faa05334ab2a7e4764ba03d4805, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733811349484 2024-12-10T06:15:53,195 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting edaaf69ad8574c8384190685561b0f57, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733811349883 2024-12-10T06:15:53,195 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ae64938d619d4ba8a19cdf2c9bc59ae8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733811349883 2024-12-10T06:15:53,195 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2978ab31d5534dacbde605ccfc149361, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733811352258 2024-12-10T06:15:53,195 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting b79bf70140c5466dbb8032a4832cc82d, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733811352204 2024-12-10T06:15:53,203 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:53,204 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d53961c6cb7649ee8ba709c8769acfb4 is 50, key is test_row_0/B:col10/1733811352270/Put/seqid=0 2024-12-10T06:15:53,205 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:53,206 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121037f538038da847d797f55ac26c955a21_1c35e623ec588c4a8cce9c8097008587 store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:53,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742231_1407 (size=12697) 2024-12-10T06:15:53,208 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121037f538038da847d797f55ac26c955a21_1c35e623ec588c4a8cce9c8097008587, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:53,208 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121037f538038da847d797f55ac26c955a21_1c35e623ec588c4a8cce9c8097008587 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742232_1408 (size=4469) 2024-12-10T06:15:53,612 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d53961c6cb7649ee8ba709c8769acfb4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d53961c6cb7649ee8ba709c8769acfb4 2024-12-10T06:15:53,613 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:53,613 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/dd0158addc9b415189bae9bf9d91bfd2 is 175, key is test_row_0/A:col10/1733811352270/Put/seqid=0 2024-12-10T06:15:53,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742233_1409 (size=31651) 2024-12-10T06:15:53,620 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/B of 1c35e623ec588c4a8cce9c8097008587 into d53961c6cb7649ee8ba709c8769acfb4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:53,620 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:53,620 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/B, priority=13, startTime=1733811353192; duration=0sec 2024-12-10T06:15:53,620 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:53,620 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:B 2024-12-10T06:15:53,620 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:15:53,625 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:15:53,625 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/C is initiating minor compaction (all files) 2024-12-10T06:15:53,625 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/C in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:53,625 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/09e9d7ea2f9a4062a65acc9c63c12805, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/9e8a04ad8ed94970918cc5c004ffad62, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a1d4bb645e8c441799d387d209d4921c] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=33.7 K 2024-12-10T06:15:53,626 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 09e9d7ea2f9a4062a65acc9c63c12805, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733811349484 2024-12-10T06:15:53,626 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/dd0158addc9b415189bae9bf9d91bfd2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/dd0158addc9b415189bae9bf9d91bfd2 2024-12-10T06:15:53,626 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e8a04ad8ed94970918cc5c004ffad62, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733811349883 2024-12-10T06:15:53,626 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting a1d4bb645e8c441799d387d209d4921c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733811352258 2024-12-10T06:15:53,631 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/A of 1c35e623ec588c4a8cce9c8097008587 into dd0158addc9b415189bae9bf9d91bfd2(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:53,631 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:53,631 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/A, priority=13, startTime=1733811353192; duration=0sec 2024-12-10T06:15:53,631 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:53,631 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:A 2024-12-10T06:15:53,634 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:53,635 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/909677f3398447658d037bfe6ea7e89f is 50, key is test_row_0/C:col10/1733811352270/Put/seqid=0 2024-12-10T06:15:53,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742234_1410 (size=12697) 2024-12-10T06:15:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:53,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-10T06:15:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:15:53,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105c9ab82386db463e86d0cc77cbe67526_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811353662/Put/seqid=0 2024-12-10T06:15:53,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742235_1411 (size=14994) 2024-12-10T06:15:53,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811413695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:53,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811413696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:53,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811413805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811413815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T06:15:53,936 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:53,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-10T06:15:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T06:15:53,938 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:53,939 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:53,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:54,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811414025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T06:15:54,046 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/909677f3398447658d037bfe6ea7e89f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/909677f3398447658d037bfe6ea7e89f 2024-12-10T06:15:54,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811414036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:54,052 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/C of 1c35e623ec588c4a8cce9c8097008587 into 909677f3398447658d037bfe6ea7e89f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:54,052 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:54,052 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/C, priority=13, startTime=1733811353192; duration=0sec 2024-12-10T06:15:54,053 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:54,053 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:C 2024-12-10T06:15:54,083 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:54,086 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105c9ab82386db463e86d0cc77cbe67526_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105c9ab82386db463e86d0cc77cbe67526_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:54,087 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/d454dba5ea524eb09f1d8e4757bc60d1, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:54,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/d454dba5ea524eb09f1d8e4757bc60d1 is 175, key is test_row_0/A:col10/1733811353662/Put/seqid=0 2024-12-10T06:15:54,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742236_1412 (size=39949) 2024-12-10T06:15:54,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-10T06:15:54,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/d454dba5ea524eb09f1d8e4757bc60d1 2024-12-10T06:15:54,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:54,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d90da85170724ea9ae5bcb2402494d43 is 50, key is test_row_0/B:col10/1733811353662/Put/seqid=0 2024-12-10T06:15:54,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742237_1413 (size=12301) 2024-12-10T06:15:54,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-10T06:15:54,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:54,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T06:15:54,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811414345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:54,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811414356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:54,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-10T06:15:54,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:54,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d90da85170724ea9ae5bcb2402494d43 2024-12-10T06:15:54,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/fa4023caad9b4e77abf51abc6126d185 is 50, key is test_row_0/C:col10/1733811353662/Put/seqid=0 2024-12-10T06:15:54,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742238_1414 (size=12301) 2024-12-10T06:15:54,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-10T06:15:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T06:15:54,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-10T06:15:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-10T06:15:54,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:54,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:54,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:15:54,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811414865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811414876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:54,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/fa4023caad9b4e77abf51abc6126d185 2024-12-10T06:15:54,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/d454dba5ea524eb09f1d8e4757bc60d1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d454dba5ea524eb09f1d8e4757bc60d1 2024-12-10T06:15:54,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d454dba5ea524eb09f1d8e4757bc60d1, entries=200, sequenceid=263, filesize=39.0 K 2024-12-10T06:15:54,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d90da85170724ea9ae5bcb2402494d43 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d90da85170724ea9ae5bcb2402494d43 2024-12-10T06:15:54,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d90da85170724ea9ae5bcb2402494d43, entries=150, sequenceid=263, filesize=12.0 K 2024-12-10T06:15:54,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/fa4023caad9b4e77abf51abc6126d185 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/fa4023caad9b4e77abf51abc6126d185 2024-12-10T06:15:54,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/fa4023caad9b4e77abf51abc6126d185, entries=150, sequenceid=263, filesize=12.0 K 2024-12-10T06:15:54,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 1c35e623ec588c4a8cce9c8097008587 in 1272ms, sequenceid=263, compaction requested=false 2024-12-10T06:15:54,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:55,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-10T06:15:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:55,012 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T06:15:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:55,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210afb583c223e34338acd99d933d0d6edf_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811353694/Put/seqid=0 2024-12-10T06:15:55,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742239_1415 (size=12454) 2024-12-10T06:15:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:55,035 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210afb583c223e34338acd99d933d0d6edf_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210afb583c223e34338acd99d933d0d6edf_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/d50829a0b1b1466e93fae7e6877cb3e7, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/d50829a0b1b1466e93fae7e6877cb3e7 is 175, key is test_row_0/A:col10/1733811353694/Put/seqid=0 2024-12-10T06:15:55,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742240_1416 (size=31255) 2024-12-10T06:15:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T06:15:55,442 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/d50829a0b1b1466e93fae7e6877cb3e7 2024-12-10T06:15:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/1efd144bb0de4811b8df4ba726b00b2a is 50, key is test_row_0/B:col10/1733811353694/Put/seqid=0 2024-12-10T06:15:55,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742241_1417 (size=12301) 2024-12-10T06:15:55,855 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/1efd144bb0de4811b8df4ba726b00b2a 2024-12-10T06:15:55,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/71b84b01006a420daec0fb6c7f4ca6d6 is 50, key is test_row_0/C:col10/1733811353694/Put/seqid=0 2024-12-10T06:15:55,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742242_1418 (size=12301) 2024-12-10T06:15:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:55,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:55,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811355955, max timestamp=1733811355981Num uniques 859; 2024-12-10T06:15:55,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:55,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811355955, max timestamp=1733811355981Num uniques 859; 2024-12-10T06:15:55,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811355955, max timestamp=1733811355981Num uniques 859; 2024-12-10T06:15:55,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:55,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T06:15:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811416091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811416097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811416215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811416215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,267 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/71b84b01006a420daec0fb6c7f4ca6d6 2024-12-10T06:15:56,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/d50829a0b1b1466e93fae7e6877cb3e7 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d50829a0b1b1466e93fae7e6877cb3e7 2024-12-10T06:15:56,279 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d50829a0b1b1466e93fae7e6877cb3e7, entries=150, sequenceid=273, filesize=30.5 K 2024-12-10T06:15:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/1efd144bb0de4811b8df4ba726b00b2a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1efd144bb0de4811b8df4ba726b00b2a 2024-12-10T06:15:56,296 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1efd144bb0de4811b8df4ba726b00b2a, entries=150, sequenceid=273, filesize=12.0 K 2024-12-10T06:15:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/71b84b01006a420daec0fb6c7f4ca6d6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/71b84b01006a420daec0fb6c7f4ca6d6 2024-12-10T06:15:56,303 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/71b84b01006a420daec0fb6c7f4ca6d6, entries=150, sequenceid=273, filesize=12.0 K 2024-12-10T06:15:56,307 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 1c35e623ec588c4a8cce9c8097008587 in 1294ms, sequenceid=273, compaction requested=true 2024-12-10T06:15:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-10T06:15:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-10T06:15:56,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-10T06:15:56,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-12-10T06:15:56,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3760 sec 2024-12-10T06:15:56,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-10T06:15:56,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:56,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T06:15:56,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:56,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T06:15:56,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:56,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T06:15:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:56,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108dbde098167c479f85b5267c5b649935_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811356095/Put/seqid=0 2024-12-10T06:15:56,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742243_1419 (size=14994) 2024-12-10T06:15:56,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811416494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811416495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811416615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811416615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811416835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811416835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:56,874 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:56,877 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108dbde098167c479f85b5267c5b649935_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108dbde098167c479f85b5267c5b649935_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:56,878 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/2c6ebff0a3e0468c8e77889dcbb5e84e, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:56,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/2c6ebff0a3e0468c8e77889dcbb5e84e is 175, key is test_row_0/A:col10/1733811356095/Put/seqid=0 2024-12-10T06:15:56,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742244_1420 (size=39949) 2024-12-10T06:15:57,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811417156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:57,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811417156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:57,283 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/2c6ebff0a3e0468c8e77889dcbb5e84e 2024-12-10T06:15:57,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/4c15961490ca4d55a6afe54d8f2c7942 is 50, key is test_row_0/B:col10/1733811356095/Put/seqid=0 2024-12-10T06:15:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742245_1421 (size=12301) 2024-12-10T06:15:57,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1733811417536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:57,550 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18339 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60368 deadline: 1733811417555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:57,567 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18357 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733811417575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:57,590 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 1c35e623ec588c4a8cce9c8097008587 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18379 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:15:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60390 deadline: 1733811417665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:57,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811417675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:57,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/4c15961490ca4d55a6afe54d8f2c7942 2024-12-10T06:15:57,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/a433f5ae7529436dae61114afd7d0654 is 50, key is test_row_0/C:col10/1733811356095/Put/seqid=0 2024-12-10T06:15:57,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742246_1422 (size=12301) 2024-12-10T06:15:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T06:15:58,105 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:15:58,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/a433f5ae7529436dae61114afd7d0654 2024-12-10T06:15:58,106 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:15:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-10T06:15:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T06:15:58,107 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:15:58,108 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:15:58,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:15:58,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/2c6ebff0a3e0468c8e77889dcbb5e84e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/2c6ebff0a3e0468c8e77889dcbb5e84e 2024-12-10T06:15:58,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/2c6ebff0a3e0468c8e77889dcbb5e84e, entries=200, sequenceid=311, filesize=39.0 K 2024-12-10T06:15:58,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/4c15961490ca4d55a6afe54d8f2c7942 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/4c15961490ca4d55a6afe54d8f2c7942 2024-12-10T06:15:58,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/4c15961490ca4d55a6afe54d8f2c7942, entries=150, sequenceid=311, filesize=12.0 K 2024-12-10T06:15:58,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/a433f5ae7529436dae61114afd7d0654 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a433f5ae7529436dae61114afd7d0654 2024-12-10T06:15:58,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a433f5ae7529436dae61114afd7d0654, entries=150, sequenceid=311, filesize=12.0 K 2024-12-10T06:15:58,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 1c35e623ec588c4a8cce9c8097008587 in 1668ms, sequenceid=311, compaction requested=true 2024-12-10T06:15:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:15:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:58,123 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:58,123 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:58,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:15:58,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:58,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:15:58,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:58,125 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142804 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:58,125 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:58,125 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/B is initiating minor compaction (all files) 2024-12-10T06:15:58,125 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/A is initiating minor compaction (all files) 2024-12-10T06:15:58,125 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/B in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:58,125 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/A in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:58,125 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d53961c6cb7649ee8ba709c8769acfb4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d90da85170724ea9ae5bcb2402494d43, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1efd144bb0de4811b8df4ba726b00b2a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/4c15961490ca4d55a6afe54d8f2c7942] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=48.4 K 2024-12-10T06:15:58,125 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/dd0158addc9b415189bae9bf9d91bfd2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d454dba5ea524eb09f1d8e4757bc60d1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d50829a0b1b1466e93fae7e6877cb3e7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/2c6ebff0a3e0468c8e77889dcbb5e84e] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=139.5 K 2024-12-10T06:15:58,125 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:58,125 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/dd0158addc9b415189bae9bf9d91bfd2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d454dba5ea524eb09f1d8e4757bc60d1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d50829a0b1b1466e93fae7e6877cb3e7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/2c6ebff0a3e0468c8e77889dcbb5e84e] 2024-12-10T06:15:58,125 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d53961c6cb7649ee8ba709c8769acfb4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733811352258 2024-12-10T06:15:58,125 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd0158addc9b415189bae9bf9d91bfd2, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733811352258 2024-12-10T06:15:58,126 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d90da85170724ea9ae5bcb2402494d43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733811352473 2024-12-10T06:15:58,126 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting d454dba5ea524eb09f1d8e4757bc60d1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733811352473 2024-12-10T06:15:58,126 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting d50829a0b1b1466e93fae7e6877cb3e7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733811353688 2024-12-10T06:15:58,126 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 1efd144bb0de4811b8df4ba726b00b2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733811353688 2024-12-10T06:15:58,126 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c6ebff0a3e0468c8e77889dcbb5e84e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733811356051 2024-12-10T06:15:58,126 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c15961490ca4d55a6afe54d8f2c7942, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733811356076 2024-12-10T06:15:58,133 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:58,134 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210656c685e061b47a293deca68f8b6fc44_1c35e623ec588c4a8cce9c8097008587 store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:58,136 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#B#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:58,137 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d5faa1623a554cbe8aec757796012521 is 50, key is test_row_0/B:col10/1733811356095/Put/seqid=0 2024-12-10T06:15:58,137 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210656c685e061b47a293deca68f8b6fc44_1c35e623ec588c4a8cce9c8097008587, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:58,138 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210656c685e061b47a293deca68f8b6fc44_1c35e623ec588c4a8cce9c8097008587 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:58,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742248_1424 (size=4469) 2024-12-10T06:15:58,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742247_1423 (size=12983) 2024-12-10T06:15:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T06:15:58,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T06:15:58,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:58,262 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T06:15:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:15:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:15:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:15:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:15:58,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107c10d673c75e47c4bc0efd334a6f38c8_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811356493/Put/seqid=0 2024-12-10T06:15:58,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742249_1425 (size=12454) 2024-12-10T06:15:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T06:15:58,549 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#A#compaction#351 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:58,550 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/208eab1f39ce4c1d83f222a802cf566a is 175, key is test_row_0/A:col10/1733811356095/Put/seqid=0 2024-12-10T06:15:58,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742250_1426 (size=31937) 2024-12-10T06:15:58,557 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/d5faa1623a554cbe8aec757796012521 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d5faa1623a554cbe8aec757796012521 2024-12-10T06:15:58,563 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/B of 1c35e623ec588c4a8cce9c8097008587 into d5faa1623a554cbe8aec757796012521(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:58,563 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:58,563 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/B, priority=12, startTime=1733811358123; duration=0sec 2024-12-10T06:15:58,563 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:15:58,563 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:B 2024-12-10T06:15:58,564 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:15:58,565 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/208eab1f39ce4c1d83f222a802cf566a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/208eab1f39ce4c1d83f222a802cf566a 2024-12-10T06:15:58,565 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:15:58,565 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 1c35e623ec588c4a8cce9c8097008587/C is initiating minor compaction (all files) 2024-12-10T06:15:58,565 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c35e623ec588c4a8cce9c8097008587/C in TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:58,565 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/909677f3398447658d037bfe6ea7e89f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/fa4023caad9b4e77abf51abc6126d185, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/71b84b01006a420daec0fb6c7f4ca6d6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a433f5ae7529436dae61114afd7d0654] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp, totalSize=48.4 K 2024-12-10T06:15:58,565 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 909677f3398447658d037bfe6ea7e89f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733811352258 2024-12-10T06:15:58,565 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4023caad9b4e77abf51abc6126d185, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733811352473 2024-12-10T06:15:58,566 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b84b01006a420daec0fb6c7f4ca6d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733811353688 2024-12-10T06:15:58,566 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting a433f5ae7529436dae61114afd7d0654, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733811356076 2024-12-10T06:15:58,569 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/A of 1c35e623ec588c4a8cce9c8097008587 into 208eab1f39ce4c1d83f222a802cf566a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:58,569 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:58,569 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/A, priority=12, startTime=1733811358123; duration=0sec 2024-12-10T06:15:58,570 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:58,570 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:A 2024-12-10T06:15:58,576 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c35e623ec588c4a8cce9c8097008587#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:15:58,576 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/bfbda9f73ae44c85a0847d45b9403c90 is 50, key is test_row_0/C:col10/1733811356095/Put/seqid=0 2024-12-10T06:15:58,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742251_1427 (size=12983) 2024-12-10T06:15:58,584 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/bfbda9f73ae44c85a0847d45b9403c90 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/bfbda9f73ae44c85a0847d45b9403c90 2024-12-10T06:15:58,589 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c35e623ec588c4a8cce9c8097008587/C of 1c35e623ec588c4a8cce9c8097008587 into bfbda9f73ae44c85a0847d45b9403c90(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:15:58,589 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:58,589 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., storeName=1c35e623ec588c4a8cce9c8097008587/C, priority=12, startTime=1733811358124; duration=0sec 2024-12-10T06:15:58,589 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:15:58,589 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:C 2024-12-10T06:15:58,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:15:58,677 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107c10d673c75e47c4bc0efd334a6f38c8_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107c10d673c75e47c4bc0efd334a6f38c8_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b80df9fbc90c41cd910ba523208833cb, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:15:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b80df9fbc90c41cd910ba523208833cb is 175, key is test_row_0/A:col10/1733811356493/Put/seqid=0 2024-12-10T06:15:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:15:58,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. as already flushing 2024-12-10T06:15:58,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742252_1428 (size=31255) 2024-12-10T06:15:58,683 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b80df9fbc90c41cd910ba523208833cb 2024-12-10T06:15:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/b921a19d5e0d44fe9bc7ded8e83a2e6a is 50, key is test_row_0/B:col10/1733811356493/Put/seqid=0 2024-12-10T06:15:58,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742253_1429 (size=12301) 2024-12-10T06:15:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T06:15:58,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811358676, max timestamp=1733811358757Num uniques 859; 2024-12-10T06:15:58,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:58,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811358676, max timestamp=1733811358757Num uniques 859; 2024-12-10T06:15:58,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811358676, max timestamp=1733811358757Num uniques 859; 2024-12-10T06:15:58,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:58,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:15:58,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:15:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60400 deadline: 1733811418847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:15:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:15:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:15:58,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811358826, max timestamp=1733811358861Num uniques 1068; 2024-12-10T06:15:58,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811358826, max timestamp=1733811358861Num uniques 1068; 2024-12-10T06:15:58,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T06:15:58,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T06:15:58,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733811358826, max timestamp=1733811358861Num uniques 1068; 2024-12-10T06:15:58,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T06:15:59,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733811358965, max timestamp=1733811358995Num uniques 1027; 2024-12-10T06:15:59,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733811358965, max timestamp=1733811358995Num uniques 1027; 2024-12-10T06:15:59,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-10T06:15:59,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-10T06:15:59,001 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-12-10T06:15:59,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733811358965, max timestamp=1733811358995Num uniques 1027; 2024-12-10T06:15:59,001 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:59,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-10T06:15:59,001 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,001 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,001 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:59,002 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-12-10T06:15:59,002 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:59,002 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,002 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:59,009 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-12-10T06:15:59,009 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:59,009 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,009 INFO [Thread-1573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1573. 2024-12-10T06:15:59,009 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,009 DEBUG [Thread-1573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:59,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:59,009 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,009 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:59,016 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-12-10T06:15:59,016 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:59,016 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,016 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:59,019 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-12-10T06:15:59,019 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:59,019 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,019 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:59,020 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-12-10T06:15:59,020 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:15:59,020 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,020 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:15:59,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:15:59,093 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/b921a19d5e0d44fe9bc7ded8e83a2e6a 2024-12-10T06:15:59,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/d71b51f32af24ce8a9e530faf3b174ab is 50, key is test_row_0/C:col10/1733811356493/Put/seqid=0 2024-12-10T06:15:59,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742254_1430 (size=12301) 2024-12-10T06:15:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T06:15:59,503 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/d71b51f32af24ce8a9e530faf3b174ab 2024-12-10T06:15:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/b80df9fbc90c41cd910ba523208833cb as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b80df9fbc90c41cd910ba523208833cb 2024-12-10T06:15:59,509 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b80df9fbc90c41cd910ba523208833cb, entries=150, sequenceid=319, filesize=30.5 K 2024-12-10T06:15:59,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/b921a19d5e0d44fe9bc7ded8e83a2e6a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b921a19d5e0d44fe9bc7ded8e83a2e6a 2024-12-10T06:15:59,512 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b921a19d5e0d44fe9bc7ded8e83a2e6a, entries=150, sequenceid=319, filesize=12.0 K 2024-12-10T06:15:59,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/d71b51f32af24ce8a9e530faf3b174ab as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d71b51f32af24ce8a9e530faf3b174ab 2024-12-10T06:15:59,515 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d71b51f32af24ce8a9e530faf3b174ab, entries=150, sequenceid=319, filesize=12.0 K 2024-12-10T06:15:59,516 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=46.96 KB/48090 for 1c35e623ec588c4a8cce9c8097008587 in 1254ms, sequenceid=319, compaction requested=false 2024-12-10T06:15:59,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:15:59,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:15:59,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-10T06:15:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-10T06:15:59,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-10T06:15:59,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-12-10T06:15:59,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4140 sec 2024-12-10T06:16:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T06:16:00,265 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:05,916 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T06:16:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:07,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=53.67 KB heapSize=139.55 KB 2024-12-10T06:16:07,597 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-12-10T06:16:07,597 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:07,597 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:07,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:16:07,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:07,597 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:07,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:16:07,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:07,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:07,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:16:07,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:07,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121007155777a6ab4a00a8bb425e88747757_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_0/A:col10/1733811367595/Put/seqid=0 2024-12-10T06:16:07,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742255_1431 (size=12454) 2024-12-10T06:16:07,616 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-12-10T06:16:07,616 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:07,616 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:07,617 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:07,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:07,637 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-12-10T06:16:07,637 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:07,637 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:07,637 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:07,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:07,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T06:16:07,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 154 2024-12-10T06:16:07,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-10T06:16:07,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-12-10T06:16:07,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-10T06:16:07,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-10T06:16:07,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T06:16:07,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T06:16:07,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 937 2024-12-10T06:16:07,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2808 rows 2024-12-10T06:16:07,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 937 2024-12-10T06:16:07,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2808 rows 2024-12-10T06:16:07,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 937 2024-12-10T06:16:07,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2808 rows 2024-12-10T06:16:07,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 941 2024-12-10T06:16:07,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2817 rows 2024-12-10T06:16:07,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 937 2024-12-10T06:16:07,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2811 rows 2024-12-10T06:16:07,638 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:16:07,638 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:16:07,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:07,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:07,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:07,639 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T06:16:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T06:16:07,641 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811367641"}]},"ts":"1733811367641"} 2024-12-10T06:16:07,642 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T06:16:07,642 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T06:16:07,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T06:16:07,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, UNASSIGN}] 2024-12-10T06:16:07,645 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, UNASSIGN 2024-12-10T06:16:07,645 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=1c35e623ec588c4a8cce9c8097008587, regionState=CLOSING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:07,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, UNASSIGN because future has completed 2024-12-10T06:16:07,647 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:16:07,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:16:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T06:16:07,799 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:07,799 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:16:07,799 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 1c35e623ec588c4a8cce9c8097008587, disabling compactions & flushes 2024-12-10T06:16:07,799 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:16:07,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T06:16:08,006 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:08,009 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121007155777a6ab4a00a8bb425e88747757_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121007155777a6ab4a00a8bb425e88747757_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:08,010 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/a232021c0335409980e91ba3e4b0b081, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:16:08,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/a232021c0335409980e91ba3e4b0b081 is 175, key is test_row_0/A:col10/1733811367595/Put/seqid=0 2024-12-10T06:16:08,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742256_1432 (size=31255) 2024-12-10T06:16:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T06:16:08,414 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/a232021c0335409980e91ba3e4b0b081 2024-12-10T06:16:08,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/179275d7a4d249dfb855a1a8fe0a8886 is 50, key is test_row_0/B:col10/1733811367595/Put/seqid=0 2024-12-10T06:16:08,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742257_1433 (size=12301) 2024-12-10T06:16:08,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/179275d7a4d249dfb855a1a8fe0a8886 2024-12-10T06:16:08,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/a8fab6c3b9f24ae5a675d339258fca17 is 50, key is test_row_0/C:col10/1733811367595/Put/seqid=0 2024-12-10T06:16:08,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742258_1434 (size=12301) 2024-12-10T06:16:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T06:16:08,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/a8fab6c3b9f24ae5a675d339258fca17 2024-12-10T06:16:08,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/a232021c0335409980e91ba3e4b0b081 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/a232021c0335409980e91ba3e4b0b081 2024-12-10T06:16:08,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/a232021c0335409980e91ba3e4b0b081, entries=150, sequenceid=380, filesize=30.5 K 2024-12-10T06:16:08,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/179275d7a4d249dfb855a1a8fe0a8886 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/179275d7a4d249dfb855a1a8fe0a8886 2024-12-10T06:16:08,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/179275d7a4d249dfb855a1a8fe0a8886, entries=150, sequenceid=380, filesize=12.0 K 2024-12-10T06:16:08,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/a8fab6c3b9f24ae5a675d339258fca17 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a8fab6c3b9f24ae5a675d339258fca17 2024-12-10T06:16:08,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a8fab6c3b9f24ae5a675d339258fca17, entries=150, sequenceid=380, filesize=12.0 K 2024-12-10T06:16:08,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~139.50 KB/142848, currentSize=13.42 KB/13740 for 1c35e623ec588c4a8cce9c8097008587 in 1251ms, sequenceid=380, compaction requested=true 2024-12-10T06:16:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c35e623ec588c4a8cce9c8097008587: 2024-12-10T06:16:08,847 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:16:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:08,847 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:16:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:08,847 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. because compaction request was cancelled 2024-12-10T06:16:08,847 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. after waiting 0 ms 2024-12-10T06:16:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:08,847 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:A 2024-12-10T06:16:08,847 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:16:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:08,847 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. because compaction request was cancelled 2024-12-10T06:16:08,847 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:B 2024-12-10T06:16:08,847 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. because compaction request was cancelled 2024-12-10T06:16:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c35e623ec588c4a8cce9c8097008587:C, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:08,847 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c35e623ec588c4a8cce9c8097008587:C 2024-12-10T06:16:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:08,847 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 1c35e623ec588c4a8cce9c8097008587 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-10T06:16:08,847 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=A 2024-12-10T06:16:08,847 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:08,847 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=B 2024-12-10T06:16:08,847 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:08,847 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c35e623ec588c4a8cce9c8097008587, store=C 2024-12-10T06:16:08,847 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:08,851 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121062527e01b31046019f7c4dc0d9b79705_1c35e623ec588c4a8cce9c8097008587 is 50, key is test_row_2/A:col10/1733811367636/Put/seqid=0 2024-12-10T06:16:08,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742259_1435 (size=7374) 2024-12-10T06:16:09,255 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:09,258 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121062527e01b31046019f7c4dc0d9b79705_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121062527e01b31046019f7c4dc0d9b79705_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:09,259 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/00c5a667cb6f4fa08a5d2aa3b7cea2a8, store: [table=TestAcidGuarantees family=A region=1c35e623ec588c4a8cce9c8097008587] 2024-12-10T06:16:09,259 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/00c5a667cb6f4fa08a5d2aa3b7cea2a8 is 175, key is test_row_2/A:col10/1733811367636/Put/seqid=0 2024-12-10T06:16:09,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742260_1436 (size=13865) 2024-12-10T06:16:09,663 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/00c5a667cb6f4fa08a5d2aa3b7cea2a8 2024-12-10T06:16:09,668 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/03a2fa6eaa0b4f4f83c6b111c5f8ea1e is 50, key is test_row_2/B:col10/1733811367636/Put/seqid=0 2024-12-10T06:16:09,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742261_1437 (size=7415) 2024-12-10T06:16:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T06:16:10,072 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/03a2fa6eaa0b4f4f83c6b111c5f8ea1e 2024-12-10T06:16:10,077 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/f292d3ea850247b994083d099c92b569 is 50, key is test_row_2/C:col10/1733811367636/Put/seqid=0 2024-12-10T06:16:10,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742262_1438 (size=7415) 2024-12-10T06:16:10,481 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/f292d3ea850247b994083d099c92b569 2024-12-10T06:16:10,484 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/A/00c5a667cb6f4fa08a5d2aa3b7cea2a8 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/00c5a667cb6f4fa08a5d2aa3b7cea2a8 2024-12-10T06:16:10,487 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/00c5a667cb6f4fa08a5d2aa3b7cea2a8, entries=50, sequenceid=385, filesize=13.5 K 2024-12-10T06:16:10,488 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/B/03a2fa6eaa0b4f4f83c6b111c5f8ea1e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/03a2fa6eaa0b4f4f83c6b111c5f8ea1e 2024-12-10T06:16:10,490 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/03a2fa6eaa0b4f4f83c6b111c5f8ea1e, entries=50, sequenceid=385, filesize=7.2 K 2024-12-10T06:16:10,491 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/.tmp/C/f292d3ea850247b994083d099c92b569 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/f292d3ea850247b994083d099c92b569 2024-12-10T06:16:10,493 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/f292d3ea850247b994083d099c92b569, entries=50, sequenceid=385, filesize=7.2 K 2024-12-10T06:16:10,494 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 1c35e623ec588c4a8cce9c8097008587 in 1647ms, sequenceid=385, compaction requested=true 2024-12-10T06:16:10,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/341f0e0eb6a54c4e93c31e5cc07e038c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/61381710d4024806a47185bcfecd51cb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4400611a87034b128ff263652340ce43, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/204bccc796c24ff1b04eeda379db31a3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/793565bc14b1486d8f0212621ffd6796, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4cb98493c6e24bb490cfd11824ad6168, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b8d7d6bc33ff4a22abe6fc689da10011, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/0b221344c2a24a5d990b72b4254e6379, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/65d78fdcccca4793b435b650445fb265, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/87c9a76d48db47319567239155585edb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/f435a5fd179441da9dd3ff183620b8c4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ee467c557c164e25b76c7eff850fa5d5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ec01e38e97b1406ba41cbe1ea963d827, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/eb6341c79a9340d9963e3a72f4bb08f9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/edaaf69ad8574c8384190685561b0f57, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b79bf70140c5466dbb8032a4832cc82d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/dd0158addc9b415189bae9bf9d91bfd2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d454dba5ea524eb09f1d8e4757bc60d1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d50829a0b1b1466e93fae7e6877cb3e7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/2c6ebff0a3e0468c8e77889dcbb5e84e] to archive 2024-12-10T06:16:10,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:16:10,497 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4400611a87034b128ff263652340ce43 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4400611a87034b128ff263652340ce43 2024-12-10T06:16:10,497 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/61381710d4024806a47185bcfecd51cb to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/61381710d4024806a47185bcfecd51cb 2024-12-10T06:16:10,497 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/341f0e0eb6a54c4e93c31e5cc07e038c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/341f0e0eb6a54c4e93c31e5cc07e038c 2024-12-10T06:16:10,497 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/204bccc796c24ff1b04eeda379db31a3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/204bccc796c24ff1b04eeda379db31a3 2024-12-10T06:16:10,497 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/793565bc14b1486d8f0212621ffd6796 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/793565bc14b1486d8f0212621ffd6796 2024-12-10T06:16:10,497 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4cb98493c6e24bb490cfd11824ad6168 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/4cb98493c6e24bb490cfd11824ad6168 2024-12-10T06:16:10,498 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b8d7d6bc33ff4a22abe6fc689da10011 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b8d7d6bc33ff4a22abe6fc689da10011 2024-12-10T06:16:10,498 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/0b221344c2a24a5d990b72b4254e6379 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/0b221344c2a24a5d990b72b4254e6379 2024-12-10T06:16:10,498 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/f435a5fd179441da9dd3ff183620b8c4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/f435a5fd179441da9dd3ff183620b8c4 2024-12-10T06:16:10,499 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/87c9a76d48db47319567239155585edb to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/87c9a76d48db47319567239155585edb 2024-12-10T06:16:10,499 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/65d78fdcccca4793b435b650445fb265 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/65d78fdcccca4793b435b650445fb265 2024-12-10T06:16:10,499 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ee467c557c164e25b76c7eff850fa5d5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ee467c557c164e25b76c7eff850fa5d5 2024-12-10T06:16:10,499 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ec01e38e97b1406ba41cbe1ea963d827 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/ec01e38e97b1406ba41cbe1ea963d827 2024-12-10T06:16:10,499 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/eb6341c79a9340d9963e3a72f4bb08f9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/eb6341c79a9340d9963e3a72f4bb08f9 2024-12-10T06:16:10,500 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/edaaf69ad8574c8384190685561b0f57 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/edaaf69ad8574c8384190685561b0f57 2024-12-10T06:16:10,500 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/dd0158addc9b415189bae9bf9d91bfd2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/dd0158addc9b415189bae9bf9d91bfd2 2024-12-10T06:16:10,500 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/2c6ebff0a3e0468c8e77889dcbb5e84e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/2c6ebff0a3e0468c8e77889dcbb5e84e 2024-12-10T06:16:10,500 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d454dba5ea524eb09f1d8e4757bc60d1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d454dba5ea524eb09f1d8e4757bc60d1 2024-12-10T06:16:10,500 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d50829a0b1b1466e93fae7e6877cb3e7 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/d50829a0b1b1466e93fae7e6877cb3e7 2024-12-10T06:16:10,502 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b79bf70140c5466dbb8032a4832cc82d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b79bf70140c5466dbb8032a4832cc82d 2024-12-10T06:16:10,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/f7917a425f1942069726be9438b989f5, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/850de6a8bbb44febb5da0f8e14054ae6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/6ba3317f71974d299dcfdcf95f8359d2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/00b22799ae474c29b57d89e7f55d6abd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/eb39ac186c4f4fea98ad44f4b35ec0dc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/e363efe38abc43fdbce46f76c2da93b4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/8a385fc34d0444e6948e57ef8cc85f43, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b44dcce70ca9495289f566a4f5c3f349, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/80edb539968e4b5a97ce11c37401ed47, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/abb3760d694149eba037a26d34ac8c7b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d4b63d9cc4fb446db36aba2c0081d26e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1d01b4dd0963497c82c6e81ab7c3f686, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/657f7faa05334ab2a7e4764ba03d4805, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/7e98ce8dbd994ac48e5480976d3176e6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/ae64938d619d4ba8a19cdf2c9bc59ae8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d53961c6cb7649ee8ba709c8769acfb4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/2978ab31d5534dacbde605ccfc149361, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d90da85170724ea9ae5bcb2402494d43, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1efd144bb0de4811b8df4ba726b00b2a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/4c15961490ca4d55a6afe54d8f2c7942] to archive 2024-12-10T06:16:10,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:16:10,505 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/f7917a425f1942069726be9438b989f5 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/f7917a425f1942069726be9438b989f5 2024-12-10T06:16:10,505 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/eb39ac186c4f4fea98ad44f4b35ec0dc to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/eb39ac186c4f4fea98ad44f4b35ec0dc 2024-12-10T06:16:10,505 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/6ba3317f71974d299dcfdcf95f8359d2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/6ba3317f71974d299dcfdcf95f8359d2 2024-12-10T06:16:10,506 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/00b22799ae474c29b57d89e7f55d6abd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/00b22799ae474c29b57d89e7f55d6abd 2024-12-10T06:16:10,506 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/850de6a8bbb44febb5da0f8e14054ae6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/850de6a8bbb44febb5da0f8e14054ae6 2024-12-10T06:16:10,506 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/e363efe38abc43fdbce46f76c2da93b4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/e363efe38abc43fdbce46f76c2da93b4 2024-12-10T06:16:10,506 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/8a385fc34d0444e6948e57ef8cc85f43 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/8a385fc34d0444e6948e57ef8cc85f43 2024-12-10T06:16:10,506 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b44dcce70ca9495289f566a4f5c3f349 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b44dcce70ca9495289f566a4f5c3f349 2024-12-10T06:16:10,507 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/80edb539968e4b5a97ce11c37401ed47 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/80edb539968e4b5a97ce11c37401ed47 2024-12-10T06:16:10,507 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/abb3760d694149eba037a26d34ac8c7b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/abb3760d694149eba037a26d34ac8c7b 2024-12-10T06:16:10,507 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d4b63d9cc4fb446db36aba2c0081d26e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d4b63d9cc4fb446db36aba2c0081d26e 2024-12-10T06:16:10,507 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1d01b4dd0963497c82c6e81ab7c3f686 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1d01b4dd0963497c82c6e81ab7c3f686 2024-12-10T06:16:10,507 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/657f7faa05334ab2a7e4764ba03d4805 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/657f7faa05334ab2a7e4764ba03d4805 2024-12-10T06:16:10,507 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d53961c6cb7649ee8ba709c8769acfb4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d53961c6cb7649ee8ba709c8769acfb4 2024-12-10T06:16:10,507 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/ae64938d619d4ba8a19cdf2c9bc59ae8 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/ae64938d619d4ba8a19cdf2c9bc59ae8 2024-12-10T06:16:10,507 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/7e98ce8dbd994ac48e5480976d3176e6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/7e98ce8dbd994ac48e5480976d3176e6 2024-12-10T06:16:10,509 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/2978ab31d5534dacbde605ccfc149361 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/2978ab31d5534dacbde605ccfc149361 2024-12-10T06:16:10,509 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d90da85170724ea9ae5bcb2402494d43 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d90da85170724ea9ae5bcb2402494d43 2024-12-10T06:16:10,509 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1efd144bb0de4811b8df4ba726b00b2a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/1efd144bb0de4811b8df4ba726b00b2a 2024-12-10T06:16:10,509 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/4c15961490ca4d55a6afe54d8f2c7942 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/4c15961490ca4d55a6afe54d8f2c7942 2024-12-10T06:16:10,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/120d089aba6b46378bfd8e1cc299a9c1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/cbc8b2d1f86c492789029dd488604fdd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/53c6100cdb8f48baa39b7687bf403a9a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c97a236b20704ad1a9626727a6fedae3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d438da778dfb415588ca4fcdb2269f46, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/2ef8c2930735431da280511d3fce05c7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7a2e642f0e6f47a49dff50cd911bfb10, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/0ec6bc8c5c1d4728a8399a218f4aa2d3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7236fa9b2b8d43e5847b93b3bb7f98e8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c8de26173d1241fe9d201b9a863d9463, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/ca33f96bab4a479bbc13905e7cbabb7d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d01d28ea48024e82b437dbaad2cba20b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/09e9d7ea2f9a4062a65acc9c63c12805, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/11b006ebded8400dadb9dea3a53d9624, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/9e8a04ad8ed94970918cc5c004ffad62, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/909677f3398447658d037bfe6ea7e89f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a1d4bb645e8c441799d387d209d4921c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/fa4023caad9b4e77abf51abc6126d185, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/71b84b01006a420daec0fb6c7f4ca6d6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a433f5ae7529436dae61114afd7d0654] to archive 2024-12-10T06:16:10,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:16:10,513 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d438da778dfb415588ca4fcdb2269f46 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d438da778dfb415588ca4fcdb2269f46 2024-12-10T06:16:10,513 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/cbc8b2d1f86c492789029dd488604fdd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/cbc8b2d1f86c492789029dd488604fdd 2024-12-10T06:16:10,513 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c97a236b20704ad1a9626727a6fedae3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c97a236b20704ad1a9626727a6fedae3 2024-12-10T06:16:10,513 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/120d089aba6b46378bfd8e1cc299a9c1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/120d089aba6b46378bfd8e1cc299a9c1 2024-12-10T06:16:10,513 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/0ec6bc8c5c1d4728a8399a218f4aa2d3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/0ec6bc8c5c1d4728a8399a218f4aa2d3 2024-12-10T06:16:10,513 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/2ef8c2930735431da280511d3fce05c7 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/2ef8c2930735431da280511d3fce05c7 2024-12-10T06:16:10,513 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7a2e642f0e6f47a49dff50cd911bfb10 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7a2e642f0e6f47a49dff50cd911bfb10 2024-12-10T06:16:10,513 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/53c6100cdb8f48baa39b7687bf403a9a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/53c6100cdb8f48baa39b7687bf403a9a 2024-12-10T06:16:10,514 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/ca33f96bab4a479bbc13905e7cbabb7d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/ca33f96bab4a479bbc13905e7cbabb7d 2024-12-10T06:16:10,514 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/09e9d7ea2f9a4062a65acc9c63c12805 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/09e9d7ea2f9a4062a65acc9c63c12805 2024-12-10T06:16:10,515 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/11b006ebded8400dadb9dea3a53d9624 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/11b006ebded8400dadb9dea3a53d9624 2024-12-10T06:16:10,515 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7236fa9b2b8d43e5847b93b3bb7f98e8 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/7236fa9b2b8d43e5847b93b3bb7f98e8 2024-12-10T06:16:10,515 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/9e8a04ad8ed94970918cc5c004ffad62 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/9e8a04ad8ed94970918cc5c004ffad62 2024-12-10T06:16:10,515 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d01d28ea48024e82b437dbaad2cba20b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d01d28ea48024e82b437dbaad2cba20b 2024-12-10T06:16:10,515 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/909677f3398447658d037bfe6ea7e89f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/909677f3398447658d037bfe6ea7e89f 2024-12-10T06:16:10,516 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c8de26173d1241fe9d201b9a863d9463 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/c8de26173d1241fe9d201b9a863d9463 2024-12-10T06:16:10,516 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/fa4023caad9b4e77abf51abc6126d185 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/fa4023caad9b4e77abf51abc6126d185 2024-12-10T06:16:10,516 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/71b84b01006a420daec0fb6c7f4ca6d6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/71b84b01006a420daec0fb6c7f4ca6d6 2024-12-10T06:16:10,516 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a433f5ae7529436dae61114afd7d0654 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a433f5ae7529436dae61114afd7d0654 2024-12-10T06:16:10,516 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a1d4bb645e8c441799d387d209d4921c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a1d4bb645e8c441799d387d209d4921c 2024-12-10T06:16:10,524 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/recovered.edits/388.seqid, newMaxSeqId=388, maxSeqId=4 2024-12-10T06:16:10,524 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. 2024-12-10T06:16:10,524 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 1c35e623ec588c4a8cce9c8097008587: Waiting for close lock at 1733811367799Running coprocessor pre-close hooks at 1733811367799Disabling compacts and flushes for region at 1733811367799Disabling writes for close at 1733811368847 (+1048 ms)Obtaining lock to block concurrent updates at 1733811368847Preparing flush snapshotting stores in 1c35e623ec588c4a8cce9c8097008587 at 1733811368847Finished memstore snapshotting TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733811368847Flushing stores of TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587. at 1733811368848 (+1 ms)Flushing 1c35e623ec588c4a8cce9c8097008587/A: creating writer at 1733811368848Flushing mob file 1c35e623ec588c4a8cce9c8097008587/A: appending metadata at 1733811368851 (+3 ms)Flushing mob file 1c35e623ec588c4a8cce9c8097008587/A: closing flushed file at 1733811368851Flushing 1c35e623ec588c4a8cce9c8097008587/A: appending metadata at 1733811369259 (+408 ms)Flushing 1c35e623ec588c4a8cce9c8097008587/A: closing flushed file at 1733811369259Flushing 1c35e623ec588c4a8cce9c8097008587/B: creating writer at 1733811369666 (+407 ms)Flushing 1c35e623ec588c4a8cce9c8097008587/B: appending metadata at 1733811369668 (+2 ms)Flushing 1c35e623ec588c4a8cce9c8097008587/B: closing flushed file at 1733811369668Flushing 1c35e623ec588c4a8cce9c8097008587/C: creating writer at 1733811370075 (+407 ms)Flushing 1c35e623ec588c4a8cce9c8097008587/C: appending metadata at 1733811370077 (+2 ms)Flushing 1c35e623ec588c4a8cce9c8097008587/C: closing flushed file at 1733811370077Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a80fdee: reopening flushed file at 1733811370484 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68c44e4f: reopening flushed file at 1733811370487 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ee3bcd2: reopening flushed file at 1733811370490 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 1c35e623ec588c4a8cce9c8097008587 in 1647ms, sequenceid=385, compaction requested=true at 1733811370494 (+4 ms)Writing region close event to WAL at 1733811370521 (+27 ms)Running coprocessor post-close hooks at 1733811370524 (+3 ms)Closed at 1733811370524 2024-12-10T06:16:10,526 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:10,526 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=1c35e623ec588c4a8cce9c8097008587, regionState=CLOSED 2024-12-10T06:16:10,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:16:10,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-10T06:16:10,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c35e623ec588c4a8cce9c8097008587, server=c42c682b34f7,43983,1733811247584 in 2.8810 sec 2024-12-10T06:16:10,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-10T06:16:10,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c35e623ec588c4a8cce9c8097008587, UNASSIGN in 2.8850 sec 2024-12-10T06:16:10,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-10T06:16:10,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8880 sec 2024-12-10T06:16:10,533 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811370533"}]},"ts":"1733811370533"} 2024-12-10T06:16:10,534 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T06:16:10,534 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T06:16:10,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.8960 sec 2024-12-10T06:16:10,584 WARN [regionserver/c42c682b34f7:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-12-10T06:16:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T06:16:11,795 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:11,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T06:16:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:11,798 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T06:16:11,798 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:11,800 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,802 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/recovered.edits] 2024-12-10T06:16:11,805 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/00c5a667cb6f4fa08a5d2aa3b7cea2a8 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/00c5a667cb6f4fa08a5d2aa3b7cea2a8 2024-12-10T06:16:11,805 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b80df9fbc90c41cd910ba523208833cb to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/b80df9fbc90c41cd910ba523208833cb 2024-12-10T06:16:11,805 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/208eab1f39ce4c1d83f222a802cf566a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/208eab1f39ce4c1d83f222a802cf566a 2024-12-10T06:16:11,805 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/a232021c0335409980e91ba3e4b0b081 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/A/a232021c0335409980e91ba3e4b0b081 2024-12-10T06:16:11,808 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/179275d7a4d249dfb855a1a8fe0a8886 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/179275d7a4d249dfb855a1a8fe0a8886 2024-12-10T06:16:11,808 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d5faa1623a554cbe8aec757796012521 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/d5faa1623a554cbe8aec757796012521 2024-12-10T06:16:11,808 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/03a2fa6eaa0b4f4f83c6b111c5f8ea1e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/03a2fa6eaa0b4f4f83c6b111c5f8ea1e 2024-12-10T06:16:11,808 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b921a19d5e0d44fe9bc7ded8e83a2e6a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/B/b921a19d5e0d44fe9bc7ded8e83a2e6a 2024-12-10T06:16:11,811 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a8fab6c3b9f24ae5a675d339258fca17 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/a8fab6c3b9f24ae5a675d339258fca17 2024-12-10T06:16:11,811 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/f292d3ea850247b994083d099c92b569 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/f292d3ea850247b994083d099c92b569 2024-12-10T06:16:11,811 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/bfbda9f73ae44c85a0847d45b9403c90 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/bfbda9f73ae44c85a0847d45b9403c90 2024-12-10T06:16:11,811 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d71b51f32af24ce8a9e530faf3b174ab to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/C/d71b51f32af24ce8a9e530faf3b174ab 2024-12-10T06:16:11,813 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/recovered.edits/388.seqid to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587/recovered.edits/388.seqid 2024-12-10T06:16:11,814 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,814 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T06:16:11,814 DEBUG [PEWorker-2 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T06:16:11,815 DEBUG [PEWorker-2 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T06:16:11,820 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121007155777a6ab4a00a8bb425e88747757_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121007155777a6ab4a00a8bb425e88747757_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,820 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100dc717e0be9047899ea3686a79f8b934_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100dc717e0be9047899ea3686a79f8b934_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,820 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103e10af84e6d84180b2b225870febd79c_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103e10af84e6d84180b2b225870febd79c_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,821 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105d04c0cdcd754b459e54dbd57ec98c0a_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105d04c0cdcd754b459e54dbd57ec98c0a_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,821 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210247673981a9342ee95de35508694688a_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210247673981a9342ee95de35508694688a_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,821 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105c9ab82386db463e86d0cc77cbe67526_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105c9ab82386db463e86d0cc77cbe67526_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,821 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210107edd5bb4d84aa7be3e66c722037a50_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210107edd5bb4d84aa7be3e66c722037a50_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,821 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121017e94ea9fde54da48df64238075f209d_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121017e94ea9fde54da48df64238075f209d_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,822 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121062527e01b31046019f7c4dc0d9b79705_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121062527e01b31046019f7c4dc0d9b79705_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,822 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210660d7831be68473a98e377e3b3bca0a1_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210660d7831be68473a98e377e3b3bca0a1_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,822 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107a7ea24c40d94f2784cddc555ff2c3cd_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107a7ea24c40d94f2784cddc555ff2c3cd_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,822 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108dbde098167c479f85b5267c5b649935_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108dbde098167c479f85b5267c5b649935_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,822 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107c10d673c75e47c4bc0efd334a6f38c8_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107c10d673c75e47c4bc0efd334a6f38c8_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,822 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a56643bf064b46f194444a20f3dee414_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a56643bf064b46f194444a20f3dee414_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,823 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210afb583c223e34338acd99d933d0d6edf_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210afb583c223e34338acd99d933d0d6edf_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,823 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d4bd70d1fc9b45c496691d5b71f43c41_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d4bd70d1fc9b45c496691d5b71f43c41_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,824 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e8a9e5acceff424daa0753c6448f4f35_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e8a9e5acceff424daa0753c6448f4f35_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,824 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e536e549c6db46ab8c50d0be28f673ea_1c35e623ec588c4a8cce9c8097008587 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e536e549c6db46ab8c50d0be28f673ea_1c35e623ec588c4a8cce9c8097008587 2024-12-10T06:16:11,824 DEBUG [PEWorker-2 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T06:16:11,826 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:11,829 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T06:16:11,831 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T06:16:11,832 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:11,832 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T06:16:11,832 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733811371832"}]},"ts":"9223372036854775807"} 2024-12-10T06:16:11,834 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T06:16:11,834 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1c35e623ec588c4a8cce9c8097008587, NAME => 'TestAcidGuarantees,,1733811335895.1c35e623ec588c4a8cce9c8097008587.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T06:16:11,834 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T06:16:11,834 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733811371834"}]},"ts":"9223372036854775807"} 2024-12-10T06:16:11,835 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T06:16:11,836 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:11,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-12-10T06:16:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T06:16:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T06:16:11,905 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:11,915 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=255 (was 256), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=389 (was 479), ProcessCount=11 (was 11), AvailableMemoryMB=6032 (was 6602) 2024-12-10T06:16:11,923 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=255, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=389, ProcessCount=11, AvailableMemoryMB=6031 2024-12-10T06:16:11,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T06:16:11,925 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:16:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:11,927 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T06:16:11,927 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:11,927 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-12-10T06:16:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-10T06:16:11,927 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T06:16:11,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742263_1439 (size=1061) 2024-12-10T06:16:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-10T06:16:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-10T06:16:12,334 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6 2024-12-10T06:16:12,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742264_1440 (size=53) 2024-12-10T06:16:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-10T06:16:12,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:16:12,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7438e40c5c6946ab37381c5eba2a38a0, disabling compactions & flushes 2024-12-10T06:16:12,739 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:12,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:12,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. after waiting 0 ms 2024-12-10T06:16:12,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:12,739 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:12,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7438e40c5c6946ab37381c5eba2a38a0: Waiting for close lock at 1733811372739Disabling compacts and flushes for region at 1733811372739Disabling writes for close at 1733811372739Writing region close event to WAL at 1733811372739Closed at 1733811372739 2024-12-10T06:16:12,740 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T06:16:12,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733811372740"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733811372740"}]},"ts":"1733811372740"} 2024-12-10T06:16:12,742 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T06:16:12,743 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T06:16:12,743 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811372743"}]},"ts":"1733811372743"} 2024-12-10T06:16:12,744 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T06:16:12,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7438e40c5c6946ab37381c5eba2a38a0, ASSIGN}] 2024-12-10T06:16:12,745 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7438e40c5c6946ab37381c5eba2a38a0, ASSIGN 2024-12-10T06:16:12,746 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7438e40c5c6946ab37381c5eba2a38a0, ASSIGN; state=OFFLINE, location=c42c682b34f7,43983,1733811247584; forceNewPlan=false, retain=false 2024-12-10T06:16:12,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=7438e40c5c6946ab37381c5eba2a38a0, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:12,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7438e40c5c6946ab37381c5eba2a38a0, ASSIGN because future has completed 2024-12-10T06:16:12,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:16:13,052 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:13,052 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:16:13,053 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,053 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:16:13,053 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,053 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,054 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,055 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:16:13,055 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7438e40c5c6946ab37381c5eba2a38a0 columnFamilyName A 2024-12-10T06:16:13,056 DEBUG [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:13,056 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] regionserver.HStore(327): Store=7438e40c5c6946ab37381c5eba2a38a0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:16:13,056 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,057 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:16:13,057 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7438e40c5c6946ab37381c5eba2a38a0 columnFamilyName B 2024-12-10T06:16:13,057 DEBUG [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:13,057 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] regionserver.HStore(327): Store=7438e40c5c6946ab37381c5eba2a38a0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:16:13,057 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,058 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:16:13,058 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7438e40c5c6946ab37381c5eba2a38a0 columnFamilyName C 2024-12-10T06:16:13,058 DEBUG [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:13,059 INFO [StoreOpener-7438e40c5c6946ab37381c5eba2a38a0-1 {}] regionserver.HStore(327): Store=7438e40c5c6946ab37381c5eba2a38a0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:16:13,059 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:13,059 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,059 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,060 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,060 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,060 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,061 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:16:13,061 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,063 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:16:13,063 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened 7438e40c5c6946ab37381c5eba2a38a0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63050353, jitterRate=-0.060476526618003845}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:16:13,063 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:13,064 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for 7438e40c5c6946ab37381c5eba2a38a0: Running coprocessor pre-open hook at 1733811373053Writing region info on filesystem at 1733811373053Initializing all the Stores at 1733811373054 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811373054Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811373054Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811373054Cleaning up temporary data from old regions at 1733811373060 (+6 ms)Running coprocessor post-open hooks at 1733811373063 (+3 ms)Region opened successfully at 1733811373063 2024-12-10T06:16:13,064 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., pid=112, masterSystemTime=1733811373050 2024-12-10T06:16:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-10T06:16:13,066 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:13,066 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:13,066 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=7438e40c5c6946ab37381c5eba2a38a0, regionState=OPEN, openSeqNum=2, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:13,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:16:13,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-10T06:16:13,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure 7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 in 170 msec 2024-12-10T06:16:13,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-10T06:16:13,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7438e40c5c6946ab37381c5eba2a38a0, ASSIGN in 326 msec 2024-12-10T06:16:13,072 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T06:16:13,072 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811373072"}]},"ts":"1733811373072"} 2024-12-10T06:16:13,073 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T06:16:13,074 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T06:16:13,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-12-10T06:16:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-10T06:16:14,075 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:14,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,077 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,077 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5e91d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,079 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776cf33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,080 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,082 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T06:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c42c682b34f7,37415,1733811246834 2024-12-10T06:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@431a8aa4 2024-12-10T06:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T06:16:14,083 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T06:16:14,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,084 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,085 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1a543c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,086 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91b8987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,087 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,088 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@714016d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,088 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7d7f9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,090 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,090 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@590d6043, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,091 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68aa6464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,093 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218036dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,094 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@479d2453, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,095 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,095 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416fffaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,096 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0a9092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,098 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8a5daf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,099 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9dd90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,100 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652f286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,101 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e96948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,103 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775708ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,104 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1604a779, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70caa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,105 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a399b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,106 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292cf75c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43602cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:14,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:14,108 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4261657a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:14,109 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:14,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4bd762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:14,110 DEBUG [Thread-1988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,110 DEBUG [Thread-1985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,110 DEBUG [Thread-1983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,110 DEBUG [Thread-1992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,110 DEBUG [Thread-1979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,110 DEBUG [Thread-1981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,111 DEBUG [Thread-1990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,111 DEBUG [Thread-1994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,112 DEBUG [Thread-1990 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,112 DEBUG [Thread-1990 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,112 DEBUG [Thread-1996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,113 DEBUG [Thread-1977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:14,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-10T06:16:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:14,118 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T06:16:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2] 2024-12-10T06:16:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:14,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T06:16:14,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:14,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:14,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:14,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:14,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:14,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:14,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/0cd82860a5d64d189a6978649200c527 is 50, key is test_row_0/A:col10/1733811374123/Put/seqid=0 2024-12-10T06:16:14,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742265_1441 (size=16681) 2024-12-10T06:16:14,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733811374128, max timestamp=1733811374160Num uniques 859; 2024-12-10T06:16:14,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-10T06:16:14,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733811374128, max timestamp=1733811374160Num uniques 859; 2024-12-10T06:16:14,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-10T06:16:14,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733811374128, max timestamp=1733811374160Num uniques 859; 2024-12-10T06:16:14,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-10T06:16:14,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811434184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811434186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811434187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811434189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811434190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T06:16:14,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T06:16:14,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811434305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811434305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811434305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811434305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811434316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T06:16:14,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:14,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T06:16:14,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811434515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811434515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811434526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811434526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811434535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/0cd82860a5d64d189a6978649200c527 2024-12-10T06:16:14,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/33b71edb75a84f429c9889bd964edac1 is 50, key is test_row_0/B:col10/1733811374123/Put/seqid=0 2024-12-10T06:16:14,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742266_1442 (size=12001) 2024-12-10T06:16:14,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T06:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T06:16:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T06:16:14,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811434835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811434835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811434836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811434836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811434855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:14,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T06:16:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:14,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:14,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/33b71edb75a84f429c9889bd964edac1 2024-12-10T06:16:15,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/733647599c5c47d988e130b5683a57bc is 50, key is test_row_0/C:col10/1733811374123/Put/seqid=0 2024-12-10T06:16:15,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742267_1443 (size=12001) 2024-12-10T06:16:15,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T06:16:15,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:15,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:15,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:15,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:15,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:15,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T06:16:15,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:15,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:15,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:15,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:15,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T06:16:15,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T06:16:15,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:15,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:15,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:15,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:15,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:15,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811435355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:15,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811435355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811435355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811435355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:15,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811435376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:15,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/733647599c5c47d988e130b5683a57bc 2024-12-10T06:16:15,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/0cd82860a5d64d189a6978649200c527 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/0cd82860a5d64d189a6978649200c527 2024-12-10T06:16:15,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/0cd82860a5d64d189a6978649200c527, entries=250, sequenceid=12, filesize=16.3 K 2024-12-10T06:16:15,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/33b71edb75a84f429c9889bd964edac1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/33b71edb75a84f429c9889bd964edac1 2024-12-10T06:16:15,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/33b71edb75a84f429c9889bd964edac1, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T06:16:15,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/733647599c5c47d988e130b5683a57bc as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/733647599c5c47d988e130b5683a57bc 2024-12-10T06:16:15,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/733647599c5c47d988e130b5683a57bc, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T06:16:15,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7438e40c5c6946ab37381c5eba2a38a0 in 1301ms, sequenceid=12, compaction requested=false 2024-12-10T06:16:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T06:16:15,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:15,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T06:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:15,498 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=134.18 KB heapSize=361.55 KB 2024-12-10T06:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:15,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/badfd0729b984aa09f399e14805f52fb is 50, key is test_row_0/A:col10/1733811374188/Put/seqid=0 2024-12-10T06:16:15,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742268_1444 (size=12001) 2024-12-10T06:16:15,913 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/badfd0729b984aa09f399e14805f52fb 2024-12-10T06:16:15,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/9da9b6ad15cb4c74977bbd06f5b6b768 is 50, key is test_row_0/B:col10/1733811374188/Put/seqid=0 2024-12-10T06:16:15,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742269_1445 (size=12001) 2024-12-10T06:16:15,949 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/9da9b6ad15cb4c74977bbd06f5b6b768 2024-12-10T06:16:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/4d82ac91924240b586a706ec76a61820 is 50, key is test_row_0/C:col10/1733811374188/Put/seqid=0 2024-12-10T06:16:15,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742270_1446 (size=12001) 2024-12-10T06:16:15,986 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/4d82ac91924240b586a706ec76a61820 2024-12-10T06:16:15,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/badfd0729b984aa09f399e14805f52fb as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/badfd0729b984aa09f399e14805f52fb 2024-12-10T06:16:15,998 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/badfd0729b984aa09f399e14805f52fb, entries=150, sequenceid=44, filesize=11.7 K 2024-12-10T06:16:16,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/9da9b6ad15cb4c74977bbd06f5b6b768 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9da9b6ad15cb4c74977bbd06f5b6b768 2024-12-10T06:16:16,016 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9da9b6ad15cb4c74977bbd06f5b6b768, entries=150, sequenceid=44, filesize=11.7 K 2024-12-10T06:16:16,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/4d82ac91924240b586a706ec76a61820 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4d82ac91924240b586a706ec76a61820 2024-12-10T06:16:16,028 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4d82ac91924240b586a706ec76a61820, entries=150, sequenceid=44, filesize=11.7 K 2024-12-10T06:16:16,030 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~361.50 KB/370176, currentSize=0 B/0 for 7438e40c5c6946ab37381c5eba2a38a0 in 531ms, sequenceid=44, compaction requested=false 2024-12-10T06:16:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-10T06:16:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-10T06:16:16,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-10T06:16:16,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-12-10T06:16:16,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.9240 sec 2024-12-10T06:16:16,042 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:16:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T06:16:16,266 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:16,267 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-10T06:16:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T06:16:16,269 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:16,270 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:16,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T06:16:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:16,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:16:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:16,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/ac5cfe22bcef420aa3aff52c8b6f843c is 50, key is test_row_0/A:col10/1733811376393/Put/seqid=0 2024-12-10T06:16:16,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742271_1447 (size=14341) 2024-12-10T06:16:16,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T06:16:16,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:16,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:16,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:16,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733811376418, max timestamp=1733811376447Num uniques 859; 2024-12-10T06:16:16,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T06:16:16,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733811376418, max timestamp=1733811376447Num uniques 859; 2024-12-10T06:16:16,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733811376418, max timestamp=1733811376447Num uniques 859; 2024-12-10T06:16:16,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T06:16:16,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T06:16:16,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811436474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811436476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811436483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811436485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811436489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T06:16:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:16,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:16,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:16,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T06:16:16,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811436595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811436595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811436616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811436616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811436616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T06:16:16,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:16,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:16,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:16,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811436805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811436805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/ac5cfe22bcef420aa3aff52c8b6f843c 2024-12-10T06:16:16,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/61fdbbc101f3430baea49ca810c0664b is 50, key is test_row_0/B:col10/1733811376393/Put/seqid=0 2024-12-10T06:16:16,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742272_1448 (size=12001) 2024-12-10T06:16:16,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/61fdbbc101f3430baea49ca810c0664b 2024-12-10T06:16:16,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/3e69841077804e658aae0eba62ed83b4 is 50, key is test_row_0/C:col10/1733811376393/Put/seqid=0 2024-12-10T06:16:16,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742273_1449 (size=12001) 2024-12-10T06:16:16,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811436835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811436835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811436835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:16,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T06:16:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:16,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T06:16:17,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T06:16:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:17,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:17,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811437116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811437116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811437155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811437155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811437155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T06:16:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:17,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:17,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/3e69841077804e658aae0eba62ed83b4 2024-12-10T06:16:17,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/ac5cfe22bcef420aa3aff52c8b6f843c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ac5cfe22bcef420aa3aff52c8b6f843c 2024-12-10T06:16:17,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ac5cfe22bcef420aa3aff52c8b6f843c, entries=200, sequenceid=55, filesize=14.0 K 2024-12-10T06:16:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/61fdbbc101f3430baea49ca810c0664b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/61fdbbc101f3430baea49ca810c0664b 2024-12-10T06:16:17,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/61fdbbc101f3430baea49ca810c0664b, entries=150, sequenceid=55, filesize=11.7 K 2024-12-10T06:16:17,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/3e69841077804e658aae0eba62ed83b4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/3e69841077804e658aae0eba62ed83b4 2024-12-10T06:16:17,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/3e69841077804e658aae0eba62ed83b4, entries=150, sequenceid=55, filesize=11.7 K 2024-12-10T06:16:17,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7438e40c5c6946ab37381c5eba2a38a0 in 863ms, sequenceid=55, compaction requested=true 2024-12-10T06:16:17,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:17,257 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:17,257 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:17,257 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:17,257 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/A is initiating minor compaction (all files) 2024-12-10T06:16:17,258 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:17,258 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/A in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:17,258 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/B is initiating minor compaction (all files) 2024-12-10T06:16:17,258 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/0cd82860a5d64d189a6978649200c527, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/badfd0729b984aa09f399e14805f52fb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ac5cfe22bcef420aa3aff52c8b6f843c] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=42.0 K 2024-12-10T06:16:17,258 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/B in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:17,258 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/33b71edb75a84f429c9889bd964edac1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9da9b6ad15cb4c74977bbd06f5b6b768, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/61fdbbc101f3430baea49ca810c0664b] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=35.2 K 2024-12-10T06:16:17,258 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cd82860a5d64d189a6978649200c527, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733811374117 2024-12-10T06:16:17,258 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting badfd0729b984aa09f399e14805f52fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733811374184 2024-12-10T06:16:17,258 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac5cfe22bcef420aa3aff52c8b6f843c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733811376388 2024-12-10T06:16:17,259 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 33b71edb75a84f429c9889bd964edac1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733811374121 2024-12-10T06:16:17,259 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da9b6ad15cb4c74977bbd06f5b6b768, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733811374184 2024-12-10T06:16:17,259 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 61fdbbc101f3430baea49ca810c0664b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733811376391 2024-12-10T06:16:17,267 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#A#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:17,267 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/74a91ea8b60c488e9075e9097349f279 is 50, key is test_row_0/A:col10/1733811376393/Put/seqid=0 2024-12-10T06:16:17,274 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#B#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:17,274 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/2938d309b8584b8fa874e13e9ce4f7bd is 50, key is test_row_0/B:col10/1733811376393/Put/seqid=0 2024-12-10T06:16:17,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742274_1450 (size=12104) 2024-12-10T06:16:17,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742275_1451 (size=12104) 2024-12-10T06:16:17,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T06:16:17,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:17,342 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-10T06:16:17,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:17,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:17,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:17,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:17,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:17,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:17,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/50b28f43b736469fbb8660f88a20f5d3 is 50, key is test_row_0/A:col10/1733811376469/Put/seqid=0 2024-12-10T06:16:17,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742276_1452 (size=12001) 2024-12-10T06:16:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T06:16:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:17,681 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/74a91ea8b60c488e9075e9097349f279 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/74a91ea8b60c488e9075e9097349f279 2024-12-10T06:16:17,686 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/A of 7438e40c5c6946ab37381c5eba2a38a0 into 74a91ea8b60c488e9075e9097349f279(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:17,686 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:17,686 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/A, priority=13, startTime=1733811377256; duration=0sec 2024-12-10T06:16:17,686 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:17,686 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:A 2024-12-10T06:16:17,686 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:17,687 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:17,687 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/C is initiating minor compaction (all files) 2024-12-10T06:16:17,687 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/C in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:17,687 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/733647599c5c47d988e130b5683a57bc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4d82ac91924240b586a706ec76a61820, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/3e69841077804e658aae0eba62ed83b4] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=35.2 K 2024-12-10T06:16:17,688 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 733647599c5c47d988e130b5683a57bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733811374121 2024-12-10T06:16:17,688 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d82ac91924240b586a706ec76a61820, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733811374184 2024-12-10T06:16:17,688 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e69841077804e658aae0eba62ed83b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733811376391 2024-12-10T06:16:17,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811437675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811437675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811437675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,693 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/2938d309b8584b8fa874e13e9ce4f7bd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2938d309b8584b8fa874e13e9ce4f7bd 2024-12-10T06:16:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811437679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,698 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/B of 7438e40c5c6946ab37381c5eba2a38a0 into 2938d309b8584b8fa874e13e9ce4f7bd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:17,698 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:17,698 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/B, priority=13, startTime=1733811377257; duration=0sec 2024-12-10T06:16:17,698 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:17,698 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:B 2024-12-10T06:16:17,699 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:17,699 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/09ee7fc74d9b44e88490c7748c7c9eae is 50, key is test_row_0/C:col10/1733811376393/Put/seqid=0 2024-12-10T06:16:17,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742277_1453 (size=12104) 2024-12-10T06:16:17,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811437692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,751 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/50b28f43b736469fbb8660f88a20f5d3 2024-12-10T06:16:17,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/8ca7a0c2f6de4c38b2c2fb8770119338 is 50, key is test_row_0/B:col10/1733811376469/Put/seqid=0 2024-12-10T06:16:17,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742278_1454 (size=12001) 2024-12-10T06:16:17,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/8ca7a0c2f6de4c38b2c2fb8770119338 2024-12-10T06:16:17,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/7ee6457b3f224814a5d53ef76049d6fa is 50, key is test_row_0/C:col10/1733811376469/Put/seqid=0 2024-12-10T06:16:17,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742279_1455 (size=12001) 2024-12-10T06:16:17,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811437805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:17,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811437815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811438025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811438025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,107 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/09ee7fc74d9b44e88490c7748c7c9eae as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/09ee7fc74d9b44e88490c7748c7c9eae 2024-12-10T06:16:18,112 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/C of 7438e40c5c6946ab37381c5eba2a38a0 into 09ee7fc74d9b44e88490c7748c7c9eae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:18,112 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:18,112 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/C, priority=13, startTime=1733811377257; duration=0sec 2024-12-10T06:16:18,112 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:18,112 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:C 2024-12-10T06:16:18,174 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/7ee6457b3f224814a5d53ef76049d6fa 2024-12-10T06:16:18,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/50b28f43b736469fbb8660f88a20f5d3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/50b28f43b736469fbb8660f88a20f5d3 2024-12-10T06:16:18,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/50b28f43b736469fbb8660f88a20f5d3, entries=150, sequenceid=89, filesize=11.7 K 2024-12-10T06:16:18,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/8ca7a0c2f6de4c38b2c2fb8770119338 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8ca7a0c2f6de4c38b2c2fb8770119338 2024-12-10T06:16:18,186 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8ca7a0c2f6de4c38b2c2fb8770119338, entries=150, sequenceid=89, filesize=11.7 K 2024-12-10T06:16:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/7ee6457b3f224814a5d53ef76049d6fa as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7ee6457b3f224814a5d53ef76049d6fa 2024-12-10T06:16:18,192 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7ee6457b3f224814a5d53ef76049d6fa, entries=150, sequenceid=89, filesize=11.7 K 2024-12-10T06:16:18,193 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 7438e40c5c6946ab37381c5eba2a38a0 in 850ms, sequenceid=89, compaction requested=false 2024-12-10T06:16:18,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:18,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:18,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-10T06:16:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-10T06:16:18,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-10T06:16:18,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9240 sec 2024-12-10T06:16:18,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.9300 sec 2024-12-10T06:16:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:18,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T06:16:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:18,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/f15a6bdb955b43e2a6215efb6394754a is 50, key is test_row_0/A:col10/1733811378350/Put/seqid=0 2024-12-10T06:16:18,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742280_1456 (size=16681) 2024-12-10T06:16:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T06:16:18,416 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:18,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-10T06:16:18,418 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T06:16:18,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:18,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:18,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811438434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811438446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T06:16:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811438555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T06:16:18,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:18,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:18,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:18,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:18,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811438565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811438705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811438705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811438705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,717 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,717 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,718 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T06:16:18,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:18,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:18,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:18,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:18,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T06:16:18,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/f15a6bdb955b43e2a6215efb6394754a 2024-12-10T06:16:18,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/c437a76ea82b419abc51dd08d31b0c80 is 50, key is test_row_0/B:col10/1733811378350/Put/seqid=0 2024-12-10T06:16:18,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742281_1457 (size=12001) 2024-12-10T06:16:18,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811438765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811438785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:18,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T06:16:18,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:18,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:18,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:18,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:18,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:19,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T06:16:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:19,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T06:16:19,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811439085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811439105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:19,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/c437a76ea82b419abc51dd08d31b0c80 2024-12-10T06:16:19,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/ae0a68e003034dbfb807a4fb74b162b0 is 50, key is test_row_0/C:col10/1733811378350/Put/seqid=0 2024-12-10T06:16:19,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T06:16:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:19,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:19,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:19,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742282_1458 (size=12001) 2024-12-10T06:16:19,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/ae0a68e003034dbfb807a4fb74b162b0 2024-12-10T06:16:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/f15a6bdb955b43e2a6215efb6394754a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/f15a6bdb955b43e2a6215efb6394754a 2024-12-10T06:16:19,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/f15a6bdb955b43e2a6215efb6394754a, entries=250, sequenceid=105, filesize=16.3 K 2024-12-10T06:16:19,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/c437a76ea82b419abc51dd08d31b0c80 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c437a76ea82b419abc51dd08d31b0c80 2024-12-10T06:16:19,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c437a76ea82b419abc51dd08d31b0c80, entries=150, sequenceid=105, filesize=11.7 K 2024-12-10T06:16:19,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/ae0a68e003034dbfb807a4fb74b162b0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/ae0a68e003034dbfb807a4fb74b162b0 2024-12-10T06:16:19,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/ae0a68e003034dbfb807a4fb74b162b0, entries=150, sequenceid=105, filesize=11.7 K 2024-12-10T06:16:19,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7438e40c5c6946ab37381c5eba2a38a0 in 859ms, sequenceid=105, compaction requested=true 2024-12-10T06:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:19,211 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:19,211 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:19,212 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:19,212 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:19,212 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/A is initiating minor compaction (all files) 2024-12-10T06:16:19,212 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/B is initiating minor compaction (all files) 2024-12-10T06:16:19,212 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/A in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:19,212 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/B in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:19,212 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/74a91ea8b60c488e9075e9097349f279, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/50b28f43b736469fbb8660f88a20f5d3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/f15a6bdb955b43e2a6215efb6394754a] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=39.8 K 2024-12-10T06:16:19,212 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2938d309b8584b8fa874e13e9ce4f7bd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8ca7a0c2f6de4c38b2c2fb8770119338, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c437a76ea82b419abc51dd08d31b0c80] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=35.3 K 2024-12-10T06:16:19,212 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a91ea8b60c488e9075e9097349f279, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733811376391 2024-12-10T06:16:19,212 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2938d309b8584b8fa874e13e9ce4f7bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733811376391 2024-12-10T06:16:19,212 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50b28f43b736469fbb8660f88a20f5d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733811376469 2024-12-10T06:16:19,213 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ca7a0c2f6de4c38b2c2fb8770119338, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733811376469 2024-12-10T06:16:19,213 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f15a6bdb955b43e2a6215efb6394754a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733811377667 2024-12-10T06:16:19,213 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting c437a76ea82b419abc51dd08d31b0c80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733811377677 2024-12-10T06:16:19,224 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:19,224 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/aafc8c7796bd45038bf29e2ae031fade is 50, key is test_row_0/B:col10/1733811378350/Put/seqid=0 2024-12-10T06:16:19,225 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:19,225 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/8fc941b7ba94402da2b748c2a1e61296 is 50, key is test_row_0/A:col10/1733811378350/Put/seqid=0 2024-12-10T06:16:19,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742283_1459 (size=12207) 2024-12-10T06:16:19,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742284_1460 (size=12207) 2024-12-10T06:16:19,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T06:16:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:19,337 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T06:16:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:19,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/ec5cf0686bd14b05ba8617b63ce1997a is 50, key is test_row_0/A:col10/1733811378429/Put/seqid=0 2024-12-10T06:16:19,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742285_1461 (size=12001) 2024-12-10T06:16:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T06:16:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:19,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:19,642 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/aafc8c7796bd45038bf29e2ae031fade as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/aafc8c7796bd45038bf29e2ae031fade 2024-12-10T06:16:19,645 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/8fc941b7ba94402da2b748c2a1e61296 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/8fc941b7ba94402da2b748c2a1e61296 2024-12-10T06:16:19,646 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/B of 7438e40c5c6946ab37381c5eba2a38a0 into aafc8c7796bd45038bf29e2ae031fade(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:19,647 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:19,647 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/B, priority=13, startTime=1733811379211; duration=0sec 2024-12-10T06:16:19,647 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:19,647 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:B 2024-12-10T06:16:19,647 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:19,648 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:19,648 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/C is initiating minor compaction (all files) 2024-12-10T06:16:19,648 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/C in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:19,648 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/09ee7fc74d9b44e88490c7748c7c9eae, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7ee6457b3f224814a5d53ef76049d6fa, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/ae0a68e003034dbfb807a4fb74b162b0] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=35.3 K 2024-12-10T06:16:19,648 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 09ee7fc74d9b44e88490c7748c7c9eae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733811376391 2024-12-10T06:16:19,650 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ee6457b3f224814a5d53ef76049d6fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733811376469 2024-12-10T06:16:19,650 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0a68e003034dbfb807a4fb74b162b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733811377677 2024-12-10T06:16:19,651 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/A of 7438e40c5c6946ab37381c5eba2a38a0 into 8fc941b7ba94402da2b748c2a1e61296(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:19,651 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:19,651 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/A, priority=13, startTime=1733811379211; duration=0sec 2024-12-10T06:16:19,651 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:19,651 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:A 2024-12-10T06:16:19,656 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#C#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:19,657 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/de9b6b27bf254439a176d1c70ba4991b is 50, key is test_row_0/C:col10/1733811378350/Put/seqid=0 2024-12-10T06:16:19,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742286_1462 (size=12207) 2024-12-10T06:16:19,668 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/de9b6b27bf254439a176d1c70ba4991b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/de9b6b27bf254439a176d1c70ba4991b 2024-12-10T06:16:19,671 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/C of 7438e40c5c6946ab37381c5eba2a38a0 into de9b6b27bf254439a176d1c70ba4991b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:19,671 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:19,671 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/C, priority=13, startTime=1733811379211; duration=0sec 2024-12-10T06:16:19,671 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:19,671 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:C 2024-12-10T06:16:19,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811439662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:19,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811439663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:19,745 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/ec5cf0686bd14b05ba8617b63ce1997a 2024-12-10T06:16:19,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/e8883884e4674afe95782b2d49310da3 is 50, key is test_row_0/B:col10/1733811378429/Put/seqid=0 2024-12-10T06:16:19,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742287_1463 (size=12001) 2024-12-10T06:16:19,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811439785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:19,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811439785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811440005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811440005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/e8883884e4674afe95782b2d49310da3 2024-12-10T06:16:20,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/574c4bfa98164553b0e4e6fb30b7a99c is 50, key is test_row_0/C:col10/1733811378429/Put/seqid=0 2024-12-10T06:16:20,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742288_1464 (size=12001) 2024-12-10T06:16:20,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811440315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811440315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T06:16:20,569 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/574c4bfa98164553b0e4e6fb30b7a99c 2024-12-10T06:16:20,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/ec5cf0686bd14b05ba8617b63ce1997a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ec5cf0686bd14b05ba8617b63ce1997a 2024-12-10T06:16:20,576 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ec5cf0686bd14b05ba8617b63ce1997a, entries=150, sequenceid=124, filesize=11.7 K 2024-12-10T06:16:20,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/e8883884e4674afe95782b2d49310da3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/e8883884e4674afe95782b2d49310da3 2024-12-10T06:16:20,580 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/e8883884e4674afe95782b2d49310da3, entries=150, sequenceid=124, filesize=11.7 K 2024-12-10T06:16:20,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/574c4bfa98164553b0e4e6fb30b7a99c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/574c4bfa98164553b0e4e6fb30b7a99c 2024-12-10T06:16:20,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/574c4bfa98164553b0e4e6fb30b7a99c, entries=150, sequenceid=124, filesize=11.7 K 2024-12-10T06:16:20,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7438e40c5c6946ab37381c5eba2a38a0 in 1247ms, sequenceid=124, compaction requested=false 2024-12-10T06:16:20,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:20,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:20,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-10T06:16:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-10T06:16:20,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-10T06:16:20,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1670 sec 2024-12-10T06:16:20,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.1720 sec 2024-12-10T06:16:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:20,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T06:16:20,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:20,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:20,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:20,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:20,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:20,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:20,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/49674dd6a7154e3ab94f24f3376cd4dd is 50, key is test_row_0/A:col10/1733811380730/Put/seqid=0 2024-12-10T06:16:20,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742289_1465 (size=14541) 2024-12-10T06:16:20,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811440779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811440786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811440789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811440825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811440825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811440895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811440905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811440915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811441115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811441125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811441136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/49674dd6a7154e3ab94f24f3376cd4dd 2024-12-10T06:16:21,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/67f4418300b8489cbc650361f47c7a9e is 50, key is test_row_0/B:col10/1733811380730/Put/seqid=0 2024-12-10T06:16:21,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742290_1466 (size=12151) 2024-12-10T06:16:21,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811441435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811441445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811441455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/67f4418300b8489cbc650361f47c7a9e 2024-12-10T06:16:21,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/0d5387196bd14c85a2352d927fe16eb2 is 50, key is test_row_0/C:col10/1733811380730/Put/seqid=0 2024-12-10T06:16:21,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742291_1467 (size=12151) 2024-12-10T06:16:21,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811441845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,856 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811441855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,866 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/0d5387196bd14c85a2352d927fe16eb2 2024-12-10T06:16:21,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811441955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/49674dd6a7154e3ab94f24f3376cd4dd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49674dd6a7154e3ab94f24f3376cd4dd 2024-12-10T06:16:21,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811441965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:21,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49674dd6a7154e3ab94f24f3376cd4dd, entries=200, sequenceid=141, filesize=14.2 K 2024-12-10T06:16:21,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/67f4418300b8489cbc650361f47c7a9e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/67f4418300b8489cbc650361f47c7a9e 2024-12-10T06:16:21,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/67f4418300b8489cbc650361f47c7a9e, entries=150, sequenceid=141, filesize=11.9 K 2024-12-10T06:16:21,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/0d5387196bd14c85a2352d927fe16eb2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/0d5387196bd14c85a2352d927fe16eb2 2024-12-10T06:16:21,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/0d5387196bd14c85a2352d927fe16eb2, entries=150, sequenceid=141, filesize=11.9 K 2024-12-10T06:16:21,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7438e40c5c6946ab37381c5eba2a38a0 in 1252ms, sequenceid=141, compaction requested=true 2024-12-10T06:16:21,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:21,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:21,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:21,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:21,984 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:21,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:21,984 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:21,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:21,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:21,984 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:21,984 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:21,985 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/A is initiating minor compaction (all files) 2024-12-10T06:16:21,985 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/B is initiating minor compaction (all files) 2024-12-10T06:16:21,985 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/A in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:21,985 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/B in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:21,985 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/8fc941b7ba94402da2b748c2a1e61296, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ec5cf0686bd14b05ba8617b63ce1997a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49674dd6a7154e3ab94f24f3376cd4dd] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=37.8 K 2024-12-10T06:16:21,985 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/aafc8c7796bd45038bf29e2ae031fade, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/e8883884e4674afe95782b2d49310da3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/67f4418300b8489cbc650361f47c7a9e] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=35.5 K 2024-12-10T06:16:21,985 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc941b7ba94402da2b748c2a1e61296, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733811377677 2024-12-10T06:16:21,985 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting aafc8c7796bd45038bf29e2ae031fade, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733811377677 2024-12-10T06:16:21,986 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting e8883884e4674afe95782b2d49310da3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733811378429 2024-12-10T06:16:21,986 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec5cf0686bd14b05ba8617b63ce1997a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733811378429 2024-12-10T06:16:21,986 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f4418300b8489cbc650361f47c7a9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811379634 2024-12-10T06:16:21,986 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49674dd6a7154e3ab94f24f3376cd4dd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811379634 2024-12-10T06:16:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:21,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T06:16:21,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:21,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:21,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:21,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:21,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:21,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:21,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/49c2a66593454dfea84c99cc15e941c9 is 50, key is test_row_0/A:col10/1733811380787/Put/seqid=0 2024-12-10T06:16:21,993 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:21,993 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/f178a28d0606497a978ee21f83314838 is 50, key is test_row_0/B:col10/1733811380730/Put/seqid=0 2024-12-10T06:16:21,995 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#A#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:21,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742292_1468 (size=14541) 2024-12-10T06:16:21,995 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/6a822d17daaa45409228c48dfd02f940 is 50, key is test_row_0/A:col10/1733811380730/Put/seqid=0 2024-12-10T06:16:21,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/49c2a66593454dfea84c99cc15e941c9 2024-12-10T06:16:21,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742293_1469 (size=12459) 2024-12-10T06:16:22,004 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/f178a28d0606497a978ee21f83314838 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/f178a28d0606497a978ee21f83314838 2024-12-10T06:16:22,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742294_1470 (size=12459) 2024-12-10T06:16:22,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/6ef8a4b9cb214bddb6ff69d0a9df9a48 is 50, key is test_row_0/B:col10/1733811380787/Put/seqid=0 2024-12-10T06:16:22,013 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/B of 7438e40c5c6946ab37381c5eba2a38a0 into f178a28d0606497a978ee21f83314838(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:22,014 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:22,014 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/B, priority=13, startTime=1733811381984; duration=0sec 2024-12-10T06:16:22,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742295_1471 (size=12151) 2024-12-10T06:16:22,014 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:22,014 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:B 2024-12-10T06:16:22,014 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:22,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/6ef8a4b9cb214bddb6ff69d0a9df9a48 2024-12-10T06:16:22,022 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:22,022 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/C is initiating minor compaction (all files) 2024-12-10T06:16:22,022 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/C in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:22,022 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/de9b6b27bf254439a176d1c70ba4991b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/574c4bfa98164553b0e4e6fb30b7a99c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/0d5387196bd14c85a2352d927fe16eb2] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=35.5 K 2024-12-10T06:16:22,023 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/6a822d17daaa45409228c48dfd02f940 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/6a822d17daaa45409228c48dfd02f940 2024-12-10T06:16:22,023 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting de9b6b27bf254439a176d1c70ba4991b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733811377677 2024-12-10T06:16:22,023 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 574c4bfa98164553b0e4e6fb30b7a99c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733811378429 2024-12-10T06:16:22,024 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d5387196bd14c85a2352d927fe16eb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811379634 2024-12-10T06:16:22,027 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/A of 7438e40c5c6946ab37381c5eba2a38a0 into 6a822d17daaa45409228c48dfd02f940(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:22,028 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:22,028 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/A, priority=13, startTime=1733811381983; duration=0sec 2024-12-10T06:16:22,028 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:22,028 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:A 2024-12-10T06:16:22,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/cfa1f7df17d74414a0e1d7ad919e50f8 is 50, key is test_row_0/C:col10/1733811380787/Put/seqid=0 2024-12-10T06:16:22,033 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#C#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:22,033 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/808ddbe6854241ac8acce439a74dc19e is 50, key is test_row_0/C:col10/1733811380730/Put/seqid=0 2024-12-10T06:16:22,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742297_1473 (size=12459) 2024-12-10T06:16:22,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742296_1472 (size=12151) 2024-12-10T06:16:22,051 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/808ddbe6854241ac8acce439a74dc19e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/808ddbe6854241ac8acce439a74dc19e 2024-12-10T06:16:22,055 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/C of 7438e40c5c6946ab37381c5eba2a38a0 into 808ddbe6854241ac8acce439a74dc19e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:22,055 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:22,055 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/C, priority=13, startTime=1733811381984; duration=0sec 2024-12-10T06:16:22,055 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:22,055 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:C 2024-12-10T06:16:22,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811442100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:22,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811442215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:22,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811442435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:22,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/cfa1f7df17d74414a0e1d7ad919e50f8 2024-12-10T06:16:22,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/49c2a66593454dfea84c99cc15e941c9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49c2a66593454dfea84c99cc15e941c9 2024-12-10T06:16:22,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49c2a66593454dfea84c99cc15e941c9, entries=200, sequenceid=160, filesize=14.2 K 2024-12-10T06:16:22,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/6ef8a4b9cb214bddb6ff69d0a9df9a48 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6ef8a4b9cb214bddb6ff69d0a9df9a48 2024-12-10T06:16:22,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6ef8a4b9cb214bddb6ff69d0a9df9a48, entries=150, sequenceid=160, filesize=11.9 K 2024-12-10T06:16:22,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/cfa1f7df17d74414a0e1d7ad919e50f8 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/cfa1f7df17d74414a0e1d7ad919e50f8 2024-12-10T06:16:22,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/cfa1f7df17d74414a0e1d7ad919e50f8, entries=150, sequenceid=160, filesize=11.9 K 2024-12-10T06:16:22,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7438e40c5c6946ab37381c5eba2a38a0 in 477ms, sequenceid=160, compaction requested=false 2024-12-10T06:16:22,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T06:16:22,586 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:22,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-10T06:16:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T06:16:22,588 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:22,589 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:22,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T06:16:22,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T06:16:22,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:22,741 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T06:16:22,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:22,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:22,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:22,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:22,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:22,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/7c89c3044aa645bf96965eba2cfa8601 is 50, key is test_row_0/A:col10/1733811382087/Put/seqid=0 2024-12-10T06:16:22,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742298_1474 (size=12151) 2024-12-10T06:16:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:22,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T06:16:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811442908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:22,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811442975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:22,986 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:22,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811442985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:22,998 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:23,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811443025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:23,151 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/7c89c3044aa645bf96965eba2cfa8601 2024-12-10T06:16:23,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/2457134b7c2b424fb3fc838fa59e0f3e is 50, key is test_row_0/B:col10/1733811382087/Put/seqid=0 2024-12-10T06:16:23,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742299_1475 (size=12151) 2024-12-10T06:16:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T06:16:23,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811443245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:23,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/2457134b7c2b424fb3fc838fa59e0f3e 2024-12-10T06:16:23,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/b74281b9edd64ab6871cf61758b926a2 is 50, key is test_row_0/C:col10/1733811382087/Put/seqid=0 2024-12-10T06:16:23,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742300_1476 (size=12151) 2024-12-10T06:16:23,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811443565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T06:16:23,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811443865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:23,876 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:23,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:23,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811443886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:23,891 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:23,973 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/b74281b9edd64ab6871cf61758b926a2 2024-12-10T06:16:23,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/7c89c3044aa645bf96965eba2cfa8601 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/7c89c3044aa645bf96965eba2cfa8601 2024-12-10T06:16:23,980 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/7c89c3044aa645bf96965eba2cfa8601, entries=150, sequenceid=176, filesize=11.9 K 2024-12-10T06:16:23,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/2457134b7c2b424fb3fc838fa59e0f3e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2457134b7c2b424fb3fc838fa59e0f3e 2024-12-10T06:16:23,984 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2457134b7c2b424fb3fc838fa59e0f3e, entries=150, sequenceid=176, filesize=11.9 K 2024-12-10T06:16:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/b74281b9edd64ab6871cf61758b926a2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/b74281b9edd64ab6871cf61758b926a2 2024-12-10T06:16:23,987 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/b74281b9edd64ab6871cf61758b926a2, entries=150, sequenceid=176, filesize=11.9 K 2024-12-10T06:16:23,988 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7438e40c5c6946ab37381c5eba2a38a0 in 1247ms, sequenceid=176, compaction requested=true 2024-12-10T06:16:23,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:23,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:23,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-10T06:16:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-10T06:16:23,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-10T06:16:23,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-12-10T06:16:23,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4060 sec 2024-12-10T06:16:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:24,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T06:16:24,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:24,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:24,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:24,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:24,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:24,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:24,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/faadc946fdca4d0da3e7ef9608022142 is 50, key is test_row_0/A:col10/1733811382906/Put/seqid=0 2024-12-10T06:16:24,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742301_1477 (size=14541) 2024-12-10T06:16:24,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811444167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:24,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811444285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:24,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/faadc946fdca4d0da3e7ef9608022142 2024-12-10T06:16:24,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/8d372fd224ac46a4a4034b5f9b361ebb is 50, key is test_row_0/B:col10/1733811382906/Put/seqid=0 2024-12-10T06:16:24,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811444495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:24,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742302_1478 (size=12151) 2024-12-10T06:16:24,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/8d372fd224ac46a4a4034b5f9b361ebb 2024-12-10T06:16:24,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/7172ed29d4834440bd7d398de548f808 is 50, key is test_row_0/C:col10/1733811382906/Put/seqid=0 2024-12-10T06:16:24,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742303_1479 (size=12151) 2024-12-10T06:16:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T06:16:24,736 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:24,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-10T06:16:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T06:16:24,738 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:24,738 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:24,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:24,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811444815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T06:16:24,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:24,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:24,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/7172ed29d4834440bd7d398de548f808 2024-12-10T06:16:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/faadc946fdca4d0da3e7ef9608022142 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/faadc946fdca4d0da3e7ef9608022142 2024-12-10T06:16:24,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/faadc946fdca4d0da3e7ef9608022142, entries=200, sequenceid=196, filesize=14.2 K 2024-12-10T06:16:24,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/8d372fd224ac46a4a4034b5f9b361ebb as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8d372fd224ac46a4a4034b5f9b361ebb 2024-12-10T06:16:24,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8d372fd224ac46a4a4034b5f9b361ebb, entries=150, sequenceid=196, filesize=11.9 K 2024-12-10T06:16:24,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/7172ed29d4834440bd7d398de548f808 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7172ed29d4834440bd7d398de548f808 2024-12-10T06:16:24,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7172ed29d4834440bd7d398de548f808, entries=150, sequenceid=196, filesize=11.9 K 2024-12-10T06:16:24,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 7438e40c5c6946ab37381c5eba2a38a0 in 850ms, sequenceid=196, compaction requested=true 2024-12-10T06:16:24,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:24,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:24,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:24,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:24,937 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:16:24,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:24,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:24,937 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:16:24,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:24,938 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53692 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:16:24,938 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:16:24,938 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/A is initiating minor compaction (all files) 2024-12-10T06:16:24,939 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/B is initiating minor compaction (all files) 2024-12-10T06:16:24,939 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/A in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:24,939 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/B in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:24,939 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/6a822d17daaa45409228c48dfd02f940, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49c2a66593454dfea84c99cc15e941c9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/7c89c3044aa645bf96965eba2cfa8601, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/faadc946fdca4d0da3e7ef9608022142] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=52.4 K 2024-12-10T06:16:24,939 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/f178a28d0606497a978ee21f83314838, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6ef8a4b9cb214bddb6ff69d0a9df9a48, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2457134b7c2b424fb3fc838fa59e0f3e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8d372fd224ac46a4a4034b5f9b361ebb] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=47.8 K 2024-12-10T06:16:24,939 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a822d17daaa45409228c48dfd02f940, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811379634 2024-12-10T06:16:24,939 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting f178a28d0606497a978ee21f83314838, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811379634 2024-12-10T06:16:24,939 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49c2a66593454dfea84c99cc15e941c9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733811380778 2024-12-10T06:16:24,939 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef8a4b9cb214bddb6ff69d0a9df9a48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733811380785 2024-12-10T06:16:24,940 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c89c3044aa645bf96965eba2cfa8601, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733811382054 2024-12-10T06:16:24,940 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting faadc946fdca4d0da3e7ef9608022142, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733811382807 2024-12-10T06:16:24,940 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2457134b7c2b424fb3fc838fa59e0f3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733811382054 2024-12-10T06:16:24,941 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d372fd224ac46a4a4034b5f9b361ebb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733811382860 2024-12-10T06:16:24,947 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#A#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:24,948 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/1b1029c654054ec3aba25c76f3bbf15c is 50, key is test_row_0/A:col10/1733811382906/Put/seqid=0 2024-12-10T06:16:24,951 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:24,952 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/ee73cad0f834403e97f09c7907bfeaa3 is 50, key is test_row_0/B:col10/1733811382906/Put/seqid=0 2024-12-10T06:16:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742305_1481 (size=12595) 2024-12-10T06:16:24,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742304_1480 (size=12595) 2024-12-10T06:16:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:25,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T06:16:25,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:25,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:25,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:25,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:25,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:25,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:25,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/b5865dbcc1a144409e6fa2933947fbcb is 50, key is test_row_1/A:col10/1733811384150/Put/seqid=0 2024-12-10T06:16:25,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742306_1482 (size=12147) 2024-12-10T06:16:25,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T06:16:25,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811445092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:25,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811445104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:25,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:25,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811445215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:25,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811445225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:25,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811445335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:25,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:25,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:25,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,360 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/1b1029c654054ec3aba25c76f3bbf15c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1b1029c654054ec3aba25c76f3bbf15c 2024-12-10T06:16:25,360 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/ee73cad0f834403e97f09c7907bfeaa3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/ee73cad0f834403e97f09c7907bfeaa3 2024-12-10T06:16:25,364 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/B of 7438e40c5c6946ab37381c5eba2a38a0 into ee73cad0f834403e97f09c7907bfeaa3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:25,364 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:25,364 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/B, priority=12, startTime=1733811384937; duration=0sec 2024-12-10T06:16:25,364 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/A of 7438e40c5c6946ab37381c5eba2a38a0 into 1b1029c654054ec3aba25c76f3bbf15c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:25,364 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:25,364 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:B 2024-12-10T06:16:25,364 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:25,364 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/A, priority=12, startTime=1733811384937; duration=0sec 2024-12-10T06:16:25,364 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:16:25,364 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:25,364 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:A 2024-12-10T06:16:25,365 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T06:16:25,365 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/C is initiating minor compaction (all files) 2024-12-10T06:16:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T06:16:25,365 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/C in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,365 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/808ddbe6854241ac8acce439a74dc19e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/cfa1f7df17d74414a0e1d7ad919e50f8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/b74281b9edd64ab6871cf61758b926a2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7172ed29d4834440bd7d398de548f808] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=47.8 K 2024-12-10T06:16:25,366 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 808ddbe6854241ac8acce439a74dc19e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733811379634 2024-12-10T06:16:25,366 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting cfa1f7df17d74414a0e1d7ad919e50f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733811380785 2024-12-10T06:16:25,366 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b74281b9edd64ab6871cf61758b926a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733811382054 2024-12-10T06:16:25,366 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7172ed29d4834440bd7d398de548f808, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733811382860 2024-12-10T06:16:25,373 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:25,374 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/164935b4aaec49ffa640f13627d101f1 is 50, key is test_row_0/C:col10/1733811382906/Put/seqid=0 2024-12-10T06:16:25,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742307_1483 (size=12595) 2024-12-10T06:16:25,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/b5865dbcc1a144409e6fa2933947fbcb 2024-12-10T06:16:25,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/6d57aa43d4ef4c0ca8efec6749c2f382 is 50, key is test_row_1/B:col10/1733811384150/Put/seqid=0 2024-12-10T06:16:25,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742308_1484 (size=9757) 2024-12-10T06:16:25,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811445435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:25,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811445445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:25,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:25,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:25,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:25,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:25,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811445756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:25,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811445765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:25,782 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/164935b4aaec49ffa640f13627d101f1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/164935b4aaec49ffa640f13627d101f1 2024-12-10T06:16:25,787 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/C of 7438e40c5c6946ab37381c5eba2a38a0 into 164935b4aaec49ffa640f13627d101f1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:25,787 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:25,787 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/C, priority=12, startTime=1733811384937; duration=0sec 2024-12-10T06:16:25,787 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:25,787 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:C 2024-12-10T06:16:25,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:25,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/6d57aa43d4ef4c0ca8efec6749c2f382 2024-12-10T06:16:25,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/5b33e7e8d37e40b4bd7f25a86cd0eeaa is 50, key is test_row_1/C:col10/1733811384150/Put/seqid=0 2024-12-10T06:16:25,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742309_1485 (size=9757) 2024-12-10T06:16:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T06:16:25,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:25,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/5b33e7e8d37e40b4bd7f25a86cd0eeaa 2024-12-10T06:16:26,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/b5865dbcc1a144409e6fa2933947fbcb as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b5865dbcc1a144409e6fa2933947fbcb 2024-12-10T06:16:26,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b5865dbcc1a144409e6fa2933947fbcb, entries=150, sequenceid=209, filesize=11.9 K 2024-12-10T06:16:26,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/6d57aa43d4ef4c0ca8efec6749c2f382 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6d57aa43d4ef4c0ca8efec6749c2f382 2024-12-10T06:16:26,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6d57aa43d4ef4c0ca8efec6749c2f382, entries=100, sequenceid=209, filesize=9.5 K 2024-12-10T06:16:26,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/5b33e7e8d37e40b4bd7f25a86cd0eeaa as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5b33e7e8d37e40b4bd7f25a86cd0eeaa 2024-12-10T06:16:26,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5b33e7e8d37e40b4bd7f25a86cd0eeaa, entries=100, sequenceid=209, filesize=9.5 K 2024-12-10T06:16:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7438e40c5c6946ab37381c5eba2a38a0 in 1271ms, sequenceid=209, compaction requested=false 2024-12-10T06:16:26,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:26,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T06:16:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:26,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/06585a27be434116ac855b6296e55abd is 50, key is test_row_0/A:col10/1733811386271/Put/seqid=0 2024-12-10T06:16:26,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742310_1486 (size=14541) 2024-12-10T06:16:26,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/06585a27be434116ac855b6296e55abd 2024-12-10T06:16:26,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/2aef918b0a0e4027abead54fd267ea4f is 50, key is test_row_0/B:col10/1733811386271/Put/seqid=0 2024-12-10T06:16:26,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742311_1487 (size=12151) 2024-12-10T06:16:26,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811446348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:26,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811446360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:26,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:26,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811446365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:26,383 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:26,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811446465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:26,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811446475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:26,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:26,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811446685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:26,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811446695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:26,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:26,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:26,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/2aef918b0a0e4027abead54fd267ea4f 2024-12-10T06:16:26,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/88ecf4a262a04b60993f2fd1ce6a5558 is 50, key is test_row_0/C:col10/1733811386271/Put/seqid=0 2024-12-10T06:16:26,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742312_1488 (size=12151) 2024-12-10T06:16:26,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/88ecf4a262a04b60993f2fd1ce6a5558 2024-12-10T06:16:26,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/06585a27be434116ac855b6296e55abd as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/06585a27be434116ac855b6296e55abd 2024-12-10T06:16:26,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/06585a27be434116ac855b6296e55abd, entries=200, sequenceid=232, filesize=14.2 K 2024-12-10T06:16:26,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/2aef918b0a0e4027abead54fd267ea4f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2aef918b0a0e4027abead54fd267ea4f 2024-12-10T06:16:26,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2aef918b0a0e4027abead54fd267ea4f, entries=150, sequenceid=232, filesize=11.9 K 2024-12-10T06:16:26,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/88ecf4a262a04b60993f2fd1ce6a5558 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/88ecf4a262a04b60993f2fd1ce6a5558 2024-12-10T06:16:26,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/88ecf4a262a04b60993f2fd1ce6a5558, entries=150, sequenceid=232, filesize=11.9 K 2024-12-10T06:16:26,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 7438e40c5c6946ab37381c5eba2a38a0 in 515ms, sequenceid=232, compaction requested=true 2024-12-10T06:16:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:26,789 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:26,789 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:26,790 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:26,790 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:26,790 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/A is initiating minor compaction (all files) 2024-12-10T06:16:26,790 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/B is initiating minor compaction (all files) 2024-12-10T06:16:26,790 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/A in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,790 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/B in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,790 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1b1029c654054ec3aba25c76f3bbf15c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b5865dbcc1a144409e6fa2933947fbcb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/06585a27be434116ac855b6296e55abd] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=38.4 K 2024-12-10T06:16:26,790 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/ee73cad0f834403e97f09c7907bfeaa3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6d57aa43d4ef4c0ca8efec6749c2f382, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2aef918b0a0e4027abead54fd267ea4f] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=33.7 K 2024-12-10T06:16:26,791 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b1029c654054ec3aba25c76f3bbf15c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733811382860 2024-12-10T06:16:26,791 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ee73cad0f834403e97f09c7907bfeaa3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733811382860 2024-12-10T06:16:26,791 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d57aa43d4ef4c0ca8efec6749c2f382, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733811384150 2024-12-10T06:16:26,791 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5865dbcc1a144409e6fa2933947fbcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733811384150 2024-12-10T06:16:26,791 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aef918b0a0e4027abead54fd267ea4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733811385076 2024-12-10T06:16:26,791 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06585a27be434116ac855b6296e55abd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733811385076 2024-12-10T06:16:26,797 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:26,798 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/a9d82a42cf8546389914ffc0eda8cc13 is 50, key is test_row_0/B:col10/1733811386271/Put/seqid=0 2024-12-10T06:16:26,798 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#A#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:26,799 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/b20caf03d8fe4cb093bd0bedce71c1db is 50, key is test_row_0/A:col10/1733811386271/Put/seqid=0 2024-12-10T06:16:26,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742314_1490 (size=12697) 2024-12-10T06:16:26,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742313_1489 (size=12697) 2024-12-10T06:16:26,812 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/b20caf03d8fe4cb093bd0bedce71c1db as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b20caf03d8fe4cb093bd0bedce71c1db 2024-12-10T06:16:26,813 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/a9d82a42cf8546389914ffc0eda8cc13 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/a9d82a42cf8546389914ffc0eda8cc13 2024-12-10T06:16:26,817 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/A of 7438e40c5c6946ab37381c5eba2a38a0 into b20caf03d8fe4cb093bd0bedce71c1db(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:26,817 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:26,817 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/A, priority=13, startTime=1733811386789; duration=0sec 2024-12-10T06:16:26,817 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:26,817 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:A 2024-12-10T06:16:26,817 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:26,818 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/B of 7438e40c5c6946ab37381c5eba2a38a0 into a9d82a42cf8546389914ffc0eda8cc13(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:26,818 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:26,818 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/B, priority=13, startTime=1733811386789; duration=0sec 2024-12-10T06:16:26,818 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:26,818 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:B 2024-12-10T06:16:26,819 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:26,819 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/C is initiating minor compaction (all files) 2024-12-10T06:16:26,819 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/C in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,819 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/164935b4aaec49ffa640f13627d101f1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5b33e7e8d37e40b4bd7f25a86cd0eeaa, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/88ecf4a262a04b60993f2fd1ce6a5558] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=33.7 K 2024-12-10T06:16:26,819 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 164935b4aaec49ffa640f13627d101f1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733811382860 2024-12-10T06:16:26,820 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b33e7e8d37e40b4bd7f25a86cd0eeaa, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733811384150 2024-12-10T06:16:26,821 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88ecf4a262a04b60993f2fd1ce6a5558, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733811385076 2024-12-10T06:16:26,831 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:26,831 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/75926a4f5db74e5692f2d9483ad52be0 is 50, key is test_row_0/C:col10/1733811386271/Put/seqid=0 2024-12-10T06:16:26,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742315_1491 (size=12697) 2024-12-10T06:16:26,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T06:16:26,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:26,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T06:16:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T06:16:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/c01618f74a134a75a1e942d96cf1053f is 50, key is test_row_0/A:col10/1733811386357/Put/seqid=0 2024-12-10T06:16:26,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742316_1492 (size=12151) 2024-12-10T06:16:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:27,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:27,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811447084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:27,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811447086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:27,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811447195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:27,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811447205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:27,246 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/75926a4f5db74e5692f2d9483ad52be0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/75926a4f5db74e5692f2d9483ad52be0 2024-12-10T06:16:27,250 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/C of 7438e40c5c6946ab37381c5eba2a38a0 into 75926a4f5db74e5692f2d9483ad52be0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:27,250 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:27,250 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/C, priority=13, startTime=1733811386789; duration=0sec 2024-12-10T06:16:27,250 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:27,250 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:C 2024-12-10T06:16:27,300 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/c01618f74a134a75a1e942d96cf1053f 2024-12-10T06:16:27,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/c9df76e4b9774065b7615d7fa0cf99b6 is 50, key is test_row_0/B:col10/1733811386357/Put/seqid=0 2024-12-10T06:16:27,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742317_1493 (size=12151) 2024-12-10T06:16:27,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811447405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:27,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811447425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:27,710 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/c9df76e4b9774065b7615d7fa0cf99b6 2024-12-10T06:16:27,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/4f5b82f3b12f433aba570e203ffad92e is 50, key is test_row_0/C:col10/1733811386357/Put/seqid=0 2024-12-10T06:16:27,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742318_1494 (size=12151) 2024-12-10T06:16:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811447725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811447735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47622 deadline: 1733811447905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,918 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_0' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47652 deadline: 1733811447915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:27,924 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:28,121 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/4f5b82f3b12f433aba570e203ffad92e 2024-12-10T06:16:28,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/c01618f74a134a75a1e942d96cf1053f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/c01618f74a134a75a1e942d96cf1053f 2024-12-10T06:16:28,129 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/c01618f74a134a75a1e942d96cf1053f, entries=150, sequenceid=246, filesize=11.9 K 2024-12-10T06:16:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/c9df76e4b9774065b7615d7fa0cf99b6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c9df76e4b9774065b7615d7fa0cf99b6 2024-12-10T06:16:28,133 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c9df76e4b9774065b7615d7fa0cf99b6, entries=150, sequenceid=246, filesize=11.9 K 2024-12-10T06:16:28,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/4f5b82f3b12f433aba570e203ffad92e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4f5b82f3b12f433aba570e203ffad92e 2024-12-10T06:16:28,137 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4f5b82f3b12f433aba570e203ffad92e, entries=150, sequenceid=246, filesize=11.9 K 2024-12-10T06:16:28,138 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7438e40c5c6946ab37381c5eba2a38a0 in 1255ms, sequenceid=246, compaction requested=false 2024-12-10T06:16:28,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:28,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:28,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-10T06:16:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-10T06:16:28,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-10T06:16:28,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4010 sec 2024-12-10T06:16:28,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 3.4060 sec 2024-12-10T06:16:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:28,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T06:16:28,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:28,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/22505db64af04612b332442d68df4a31 is 50, key is test_row_0/A:col10/1733811388248/Put/seqid=0 2024-12-10T06:16:28,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742319_1495 (size=17181) 2024-12-10T06:16:28,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811448282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:28,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811448288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:28,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811448405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:28,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811448405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:28,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811448405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:28,420 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 7438e40c5c6946ab37381c5eba2a38a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:28,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811448625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811448625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:28,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/22505db64af04612b332442d68df4a31 2024-12-10T06:16:28,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/83443bd2114e489d84c3456ed586906c is 50, key is test_row_0/B:col10/1733811388248/Put/seqid=0 2024-12-10T06:16:28,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742320_1496 (size=12301) 2024-12-10T06:16:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T06:16:28,896 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:28,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-10T06:16:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T06:16:28,899 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:28,900 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:28,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:28,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811448935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:28,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811448935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T06:16:29,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T06:16:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:29,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/83443bd2114e489d84c3456ed586906c 2024-12-10T06:16:29,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/159e0b016d294149ad7a81dd7a3e967b is 50, key is test_row_0/C:col10/1733811388248/Put/seqid=0 2024-12-10T06:16:29,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742321_1497 (size=12301) 2024-12-10T06:16:29,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T06:16:29,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:29,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:29,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T06:16:29,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T06:16:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:29,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811449455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:29,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811449455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:29,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/159e0b016d294149ad7a81dd7a3e967b 2024-12-10T06:16:29,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/22505db64af04612b332442d68df4a31 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/22505db64af04612b332442d68df4a31 2024-12-10T06:16:29,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/22505db64af04612b332442d68df4a31, entries=250, sequenceid=269, filesize=16.8 K 2024-12-10T06:16:29,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/83443bd2114e489d84c3456ed586906c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/83443bd2114e489d84c3456ed586906c 2024-12-10T06:16:29,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/83443bd2114e489d84c3456ed586906c, entries=150, sequenceid=269, filesize=12.0 K 2024-12-10T06:16:29,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/159e0b016d294149ad7a81dd7a3e967b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/159e0b016d294149ad7a81dd7a3e967b 2024-12-10T06:16:29,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/159e0b016d294149ad7a81dd7a3e967b, entries=150, sequenceid=269, filesize=12.0 K 2024-12-10T06:16:29,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 7438e40c5c6946ab37381c5eba2a38a0 in 1249ms, sequenceid=269, compaction requested=true 2024-12-10T06:16:29,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:29,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:29,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:29,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:29,499 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:29,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:29,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:29,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:29,499 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:29,500 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:29,500 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42029 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:29,500 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/A is initiating minor compaction (all files) 2024-12-10T06:16:29,500 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/B is initiating minor compaction (all files) 2024-12-10T06:16:29,500 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/A in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,500 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/B in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,500 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b20caf03d8fe4cb093bd0bedce71c1db, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/c01618f74a134a75a1e942d96cf1053f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/22505db64af04612b332442d68df4a31] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=41.0 K 2024-12-10T06:16:29,500 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/a9d82a42cf8546389914ffc0eda8cc13, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c9df76e4b9774065b7615d7fa0cf99b6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/83443bd2114e489d84c3456ed586906c] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=36.3 K 2024-12-10T06:16:29,501 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting a9d82a42cf8546389914ffc0eda8cc13, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733811385076 2024-12-10T06:16:29,501 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting b20caf03d8fe4cb093bd0bedce71c1db, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733811385076 2024-12-10T06:16:29,501 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting c01618f74a134a75a1e942d96cf1053f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733811386297 2024-12-10T06:16:29,501 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting c9df76e4b9774065b7615d7fa0cf99b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733811386297 2024-12-10T06:16:29,502 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 83443bd2114e489d84c3456ed586906c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733811387072 2024-12-10T06:16:29,502 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22505db64af04612b332442d68df4a31, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733811387072 2024-12-10T06:16:29,509 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:29,510 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/1c53e0c6807b4866b5ad6c3e9a8acea4 is 50, key is test_row_0/A:col10/1733811388248/Put/seqid=0 2024-12-10T06:16:29,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T06:16:29,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,512 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:16:29,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:29,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:29,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:29,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:29,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:29,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:29,513 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:29,514 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/fab6ea83c0f84ab680ba6ed193003586 is 50, key is test_row_0/B:col10/1733811388248/Put/seqid=0 2024-12-10T06:16:29,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/a708ccace9424b1da139c5adb443e451 is 50, key is test_row_0/A:col10/1733811388280/Put/seqid=0 2024-12-10T06:16:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T06:16:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742322_1498 (size=12949) 2024-12-10T06:16:29,531 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/fab6ea83c0f84ab680ba6ed193003586 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/fab6ea83c0f84ab680ba6ed193003586 2024-12-10T06:16:29,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742323_1499 (size=12949) 2024-12-10T06:16:29,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742324_1500 (size=12301) 2024-12-10T06:16:29,535 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/a708ccace9424b1da139c5adb443e451 2024-12-10T06:16:29,537 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/B of 7438e40c5c6946ab37381c5eba2a38a0 into fab6ea83c0f84ab680ba6ed193003586(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:29,537 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:29,537 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/B, priority=13, startTime=1733811389499; duration=0sec 2024-12-10T06:16:29,537 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:29,537 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:B 2024-12-10T06:16:29,537 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:29,539 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:29,539 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/C is initiating minor compaction (all files) 2024-12-10T06:16:29,540 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/C in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,540 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/75926a4f5db74e5692f2d9483ad52be0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4f5b82f3b12f433aba570e203ffad92e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/159e0b016d294149ad7a81dd7a3e967b] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=36.3 K 2024-12-10T06:16:29,540 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 75926a4f5db74e5692f2d9483ad52be0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733811385076 2024-12-10T06:16:29,540 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f5b82f3b12f433aba570e203ffad92e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733811386297 2024-12-10T06:16:29,541 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 159e0b016d294149ad7a81dd7a3e967b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733811387072 2024-12-10T06:16:29,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/c2ca7348f57748abb9703f72a976c20c is 50, key is test_row_0/B:col10/1733811388280/Put/seqid=0 2024-12-10T06:16:29,548 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:29,549 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/54902d707e9c479da11914753a01578b is 50, key is test_row_0/C:col10/1733811388248/Put/seqid=0 2024-12-10T06:16:29,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742326_1502 (size=12949) 2024-12-10T06:16:29,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742325_1501 (size=12301) 2024-12-10T06:16:29,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/c2ca7348f57748abb9703f72a976c20c 2024-12-10T06:16:29,567 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/54902d707e9c479da11914753a01578b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/54902d707e9c479da11914753a01578b 2024-12-10T06:16:29,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/692595494d5b425f820b1f268d7d05ba is 50, key is test_row_0/C:col10/1733811388280/Put/seqid=0 2024-12-10T06:16:29,574 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/C of 7438e40c5c6946ab37381c5eba2a38a0 into 54902d707e9c479da11914753a01578b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:29,574 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:29,574 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/C, priority=13, startTime=1733811389499; duration=0sec 2024-12-10T06:16:29,574 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:29,574 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:C 2024-12-10T06:16:29,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742327_1503 (size=12301) 2024-12-10T06:16:29,939 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/1c53e0c6807b4866b5ad6c3e9a8acea4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1c53e0c6807b4866b5ad6c3e9a8acea4 2024-12-10T06:16:29,944 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/A of 7438e40c5c6946ab37381c5eba2a38a0 into 1c53e0c6807b4866b5ad6c3e9a8acea4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:29,944 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:29,944 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/A, priority=13, startTime=1733811389499; duration=0sec 2024-12-10T06:16:29,944 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:29,944 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:A 2024-12-10T06:16:29,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/692595494d5b425f820b1f268d7d05ba 2024-12-10T06:16:29,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/a708ccace9424b1da139c5adb443e451 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a708ccace9424b1da139c5adb443e451 2024-12-10T06:16:29,983 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a708ccace9424b1da139c5adb443e451, entries=150, sequenceid=279, filesize=12.0 K 2024-12-10T06:16:29,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/c2ca7348f57748abb9703f72a976c20c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c2ca7348f57748abb9703f72a976c20c 2024-12-10T06:16:29,988 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c2ca7348f57748abb9703f72a976c20c, entries=150, sequenceid=279, filesize=12.0 K 2024-12-10T06:16:29,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/692595494d5b425f820b1f268d7d05ba as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/692595494d5b425f820b1f268d7d05ba 2024-12-10T06:16:29,993 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/692595494d5b425f820b1f268d7d05ba, entries=150, sequenceid=279, filesize=12.0 K 2024-12-10T06:16:29,994 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 7438e40c5c6946ab37381c5eba2a38a0 in 482ms, sequenceid=279, compaction requested=false 2024-12-10T06:16:29,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:29,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:29,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-10T06:16:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-10T06:16:29,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-10T06:16:29,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0960 sec 2024-12-10T06:16:30,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.1020 sec 2024-12-10T06:16:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T06:16:30,035 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:30,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-10T06:16:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T06:16:30,038 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:30,038 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:30,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T06:16:30,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T06:16:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-10T06:16:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-10T06:16:30,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-10T06:16:30,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-10T06:16:30,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 159 msec 2024-12-10T06:16:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T06:16:30,355 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:30,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-10T06:16:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T06:16:30,358 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:30,360 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:30,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T06:16:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:30,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:16:30,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:30,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:30,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:30,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:30,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:30,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:30,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/a49106af8686462fa32245c8840a6c00 is 50, key is test_row_1/A:col10/1733811390489/Put/seqid=0 2024-12-10T06:16:30,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742328_1504 (size=14737) 2024-12-10T06:16:30,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/a49106af8686462fa32245c8840a6c00 2024-12-10T06:16:30,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/68eda72f46fc4d0ebc86fa24068aa376 is 50, key is test_row_1/B:col10/1733811390489/Put/seqid=0 2024-12-10T06:16:30,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T06:16:30,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:30,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742329_1505 (size=9857) 2024-12-10T06:16:30,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811390507, max timestamp=1733811390577Num uniques 859; 2024-12-10T06:16:30,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811390507, max timestamp=1733811390577Num uniques 859; 2024-12-10T06:16:30,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811390507, max timestamp=1733811390577Num uniques 859; 2024-12-10T06:16:30,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:16:30,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:16:30,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:16:30,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T06:16:30,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:30,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T06:16:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811450661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:30,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811450672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:30,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811450785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:30,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:30,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:30,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:30,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811450785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:30,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T06:16:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/68eda72f46fc4d0ebc86fa24068aa376 2024-12-10T06:16:30,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/9cfbbef3e7ec49d3a1fe201d4918d048 is 50, key is test_row_1/C:col10/1733811390489/Put/seqid=0 2024-12-10T06:16:30,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742330_1506 (size=9857) 2024-12-10T06:16:30,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T06:16:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:30,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T06:16:31,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811451005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:31,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811451005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:31,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T06:16:31,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:31,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:31,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:31,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:31,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:31,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T06:16:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:31,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:31,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811451325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:31,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811451326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:31,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/9cfbbef3e7ec49d3a1fe201d4918d048 2024-12-10T06:16:31,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/a49106af8686462fa32245c8840a6c00 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a49106af8686462fa32245c8840a6c00 2024-12-10T06:16:31,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a49106af8686462fa32245c8840a6c00, entries=200, sequenceid=293, filesize=14.4 K 2024-12-10T06:16:31,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/68eda72f46fc4d0ebc86fa24068aa376 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/68eda72f46fc4d0ebc86fa24068aa376 2024-12-10T06:16:31,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/68eda72f46fc4d0ebc86fa24068aa376, entries=100, sequenceid=293, filesize=9.6 K 2024-12-10T06:16:31,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/9cfbbef3e7ec49d3a1fe201d4918d048 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/9cfbbef3e7ec49d3a1fe201d4918d048 2024-12-10T06:16:31,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/9cfbbef3e7ec49d3a1fe201d4918d048, entries=100, sequenceid=293, filesize=9.6 K 2024-12-10T06:16:31,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7438e40c5c6946ab37381c5eba2a38a0 in 861ms, sequenceid=293, compaction requested=true 2024-12-10T06:16:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:31,352 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:31,352 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:31,353 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:31,353 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39987 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:31,353 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/B is initiating minor compaction (all files) 2024-12-10T06:16:31,353 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/A is initiating minor compaction (all files) 2024-12-10T06:16:31,353 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/B in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:31,353 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/A in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:31,353 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/fab6ea83c0f84ab680ba6ed193003586, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c2ca7348f57748abb9703f72a976c20c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/68eda72f46fc4d0ebc86fa24068aa376] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=34.3 K 2024-12-10T06:16:31,353 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1c53e0c6807b4866b5ad6c3e9a8acea4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a708ccace9424b1da139c5adb443e451, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a49106af8686462fa32245c8840a6c00] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=39.0 K 2024-12-10T06:16:31,353 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c53e0c6807b4866b5ad6c3e9a8acea4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733811387072 2024-12-10T06:16:31,353 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting fab6ea83c0f84ab680ba6ed193003586, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733811387072 2024-12-10T06:16:31,354 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting c2ca7348f57748abb9703f72a976c20c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733811388262 2024-12-10T06:16:31,354 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting a708ccace9424b1da139c5adb443e451, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733811388262 2024-12-10T06:16:31,354 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting a49106af8686462fa32245c8840a6c00, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733811390475 2024-12-10T06:16:31,354 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 68eda72f46fc4d0ebc86fa24068aa376, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733811390488 2024-12-10T06:16:31,360 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:31,360 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/953d337407d04d93abef92f746625e1a is 50, key is test_row_0/A:col10/1733811388280/Put/seqid=0 2024-12-10T06:16:31,362 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:31,363 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/9e670c82e8e14e3ca71aa8012ce34f7d is 50, key is test_row_0/B:col10/1733811388280/Put/seqid=0 2024-12-10T06:16:31,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742331_1507 (size=13051) 2024-12-10T06:16:31,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742332_1508 (size=13051) 2024-12-10T06:16:31,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T06:16:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:31,434 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T06:16:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:31,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/a931abdb9ac449ff8224436ea458d7c7 is 50, key is test_row_0/A:col10/1733811390632/Put/seqid=0 2024-12-10T06:16:31,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742333_1509 (size=12301) 2024-12-10T06:16:31,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/a931abdb9ac449ff8224436ea458d7c7 2024-12-10T06:16:31,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/3a44695ad502441596a00c348bb8b04d is 50, key is test_row_0/B:col10/1733811390632/Put/seqid=0 2024-12-10T06:16:31,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742334_1510 (size=12301) 2024-12-10T06:16:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T06:16:31,771 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/9e670c82e8e14e3ca71aa8012ce34f7d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9e670c82e8e14e3ca71aa8012ce34f7d 2024-12-10T06:16:31,771 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/953d337407d04d93abef92f746625e1a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/953d337407d04d93abef92f746625e1a 2024-12-10T06:16:31,776 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/B of 7438e40c5c6946ab37381c5eba2a38a0 into 9e670c82e8e14e3ca71aa8012ce34f7d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:31,776 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/A of 7438e40c5c6946ab37381c5eba2a38a0 into 953d337407d04d93abef92f746625e1a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:31,776 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:31,776 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:31,776 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/B, priority=13, startTime=1733811391352; duration=0sec 2024-12-10T06:16:31,776 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/A, priority=13, startTime=1733811391352; duration=0sec 2024-12-10T06:16:31,776 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:31,776 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:B 2024-12-10T06:16:31,776 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:31,776 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:A 2024-12-10T06:16:31,776 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:31,777 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:31,777 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/C is initiating minor compaction (all files) 2024-12-10T06:16:31,777 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/C in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:31,777 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/54902d707e9c479da11914753a01578b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/692595494d5b425f820b1f268d7d05ba, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/9cfbbef3e7ec49d3a1fe201d4918d048] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=34.3 K 2024-12-10T06:16:31,778 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 54902d707e9c479da11914753a01578b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733811387072 2024-12-10T06:16:31,778 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 692595494d5b425f820b1f268d7d05ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733811388262 2024-12-10T06:16:31,778 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cfbbef3e7ec49d3a1fe201d4918d048, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733811390488 2024-12-10T06:16:31,784 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:31,785 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/66a0850cbcda4c8194311425f747fe54 is 50, key is test_row_0/C:col10/1733811388280/Put/seqid=0 2024-12-10T06:16:31,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742335_1511 (size=13051) 2024-12-10T06:16:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:31,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:31,853 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/3a44695ad502441596a00c348bb8b04d 2024-12-10T06:16:31,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/741e416df71c4994875225bf52f42faf is 50, key is test_row_0/C:col10/1733811390632/Put/seqid=0 2024-12-10T06:16:31,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742336_1512 (size=12301) 2024-12-10T06:16:31,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:31,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811451883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811451885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811451995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811452006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,192 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/66a0850cbcda4c8194311425f747fe54 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/66a0850cbcda4c8194311425f747fe54 2024-12-10T06:16:32,196 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/C of 7438e40c5c6946ab37381c5eba2a38a0 into 66a0850cbcda4c8194311425f747fe54(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:32,196 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:32,196 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/C, priority=13, startTime=1733811391352; duration=0sec 2024-12-10T06:16:32,196 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:32,196 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:C 2024-12-10T06:16:32,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811452215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811452215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,265 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/741e416df71c4994875225bf52f42faf 2024-12-10T06:16:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/a931abdb9ac449ff8224436ea458d7c7 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a931abdb9ac449ff8224436ea458d7c7 2024-12-10T06:16:32,272 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a931abdb9ac449ff8224436ea458d7c7, entries=150, sequenceid=329, filesize=12.0 K 2024-12-10T06:16:32,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/3a44695ad502441596a00c348bb8b04d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/3a44695ad502441596a00c348bb8b04d 2024-12-10T06:16:32,276 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/3a44695ad502441596a00c348bb8b04d, entries=150, sequenceid=329, filesize=12.0 K 2024-12-10T06:16:32,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/741e416df71c4994875225bf52f42faf as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/741e416df71c4994875225bf52f42faf 2024-12-10T06:16:32,280 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/741e416df71c4994875225bf52f42faf, entries=150, sequenceid=329, filesize=12.0 K 2024-12-10T06:16:32,281 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 7438e40c5c6946ab37381c5eba2a38a0 in 847ms, sequenceid=329, compaction requested=false 2024-12-10T06:16:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-10T06:16:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-10T06:16:32,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-10T06:16:32,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9220 sec 2024-12-10T06:16:32,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.9280 sec 2024-12-10T06:16:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:32,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T06:16:32,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:32,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:32,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:32,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:32,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:32,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:32,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/bbc56781a7ba4e63aa15a0104e8b5a88 is 50, key is test_row_0/A:col10/1733811391884/Put/seqid=0 2024-12-10T06:16:32,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742337_1513 (size=12301) 2024-12-10T06:16:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T06:16:32,505 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:32,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-10T06:16:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T06:16:32,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:32,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:32,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:32,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811392552, max timestamp=1733811392576Num uniques 859; 2024-12-10T06:16:32,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811392552, max timestamp=1733811392576Num uniques 859; 2024-12-10T06:16:32,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:16:32,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:16:32,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733811392552, max timestamp=1733811392576Num uniques 859; 2024-12-10T06:16:32,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:16:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T06:16:32,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811452638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:32,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:32,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:32,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:32,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:32,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:32,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811452641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:32,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811452642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811452765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811452765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811452765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:32,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T06:16:32,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/bbc56781a7ba4e63aa15a0104e8b5a88 2024-12-10T06:16:32,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/182bd38e4c74417294a6893fc8533fa2 is 50, key is test_row_0/B:col10/1733811391884/Put/seqid=0 2024-12-10T06:16:32,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742338_1514 (size=12301) 2024-12-10T06:16:32,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:32,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:32,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:32,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:32,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:32,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:32,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811452985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811452985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811452985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:33,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:33,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:33,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T06:16:33,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/182bd38e4c74417294a6893fc8533fa2 2024-12-10T06:16:33,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/1bdcf2145234443c8ca8277cf718021c is 50, key is test_row_0/C:col10/1733811391884/Put/seqid=0 2024-12-10T06:16:33,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742339_1515 (size=12301) 2024-12-10T06:16:33,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/1bdcf2145234443c8ca8277cf718021c 2024-12-10T06:16:33,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/bbc56781a7ba4e63aa15a0104e8b5a88 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/bbc56781a7ba4e63aa15a0104e8b5a88 2024-12-10T06:16:33,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/bbc56781a7ba4e63aa15a0104e8b5a88, entries=150, sequenceid=343, filesize=12.0 K 2024-12-10T06:16:33,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/182bd38e4c74417294a6893fc8533fa2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/182bd38e4c74417294a6893fc8533fa2 2024-12-10T06:16:33,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811453305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:33,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/182bd38e4c74417294a6893fc8533fa2, entries=150, sequenceid=343, filesize=12.0 K 2024-12-10T06:16:33,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:33,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/1bdcf2145234443c8ca8277cf718021c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1bdcf2145234443c8ca8277cf718021c 2024-12-10T06:16:33,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1bdcf2145234443c8ca8277cf718021c, entries=150, sequenceid=343, filesize=12.0 K 2024-12-10T06:16:33,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7438e40c5c6946ab37381c5eba2a38a0 in 851ms, sequenceid=343, compaction requested=true 2024-12-10T06:16:33,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:33,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:33,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:33,321 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:33,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:33,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:33,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7438e40c5c6946ab37381c5eba2a38a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:33,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:33,321 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:33,322 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:33,322 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:33,322 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/A is initiating minor compaction (all files) 2024-12-10T06:16:33,322 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/B is initiating minor compaction (all files) 2024-12-10T06:16:33,322 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/B in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,322 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/A in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,322 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9e670c82e8e14e3ca71aa8012ce34f7d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/3a44695ad502441596a00c348bb8b04d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/182bd38e4c74417294a6893fc8533fa2] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=36.8 K 2024-12-10T06:16:33,322 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/953d337407d04d93abef92f746625e1a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a931abdb9ac449ff8224436ea458d7c7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/bbc56781a7ba4e63aa15a0104e8b5a88] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=36.8 K 2024-12-10T06:16:33,323 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e670c82e8e14e3ca71aa8012ce34f7d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733811388280 2024-12-10T06:16:33,323 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 953d337407d04d93abef92f746625e1a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733811388280 2024-12-10T06:16:33,323 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a44695ad502441596a00c348bb8b04d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733811390632 2024-12-10T06:16:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:33,323 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 182bd38e4c74417294a6893fc8533fa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733811391871 2024-12-10T06:16:33,323 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting a931abdb9ac449ff8224436ea458d7c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733811390632 2024-12-10T06:16:33,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T06:16:33,324 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbc56781a7ba4e63aa15a0104e8b5a88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733811391871 2024-12-10T06:16:33,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:33,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:33,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:33,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:33,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:33,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:33,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/4913e443e605461ca9b9c6b440a6410c is 50, key is test_row_0/A:col10/1733811392608/Put/seqid=0 2024-12-10T06:16:33,331 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#B#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:33,331 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#A#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:33,331 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/46f9b31a1486440b9f08bdbf52e3ebbc is 50, key is test_row_0/B:col10/1733811391884/Put/seqid=0 2024-12-10T06:16:33,331 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/1b77102fac094dc693632e2b49a782b7 is 50, key is test_row_0/A:col10/1733811391884/Put/seqid=0 2024-12-10T06:16:33,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742340_1516 (size=17181) 2024-12-10T06:16:33,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/4913e443e605461ca9b9c6b440a6410c 2024-12-10T06:16:33,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742341_1517 (size=13153) 2024-12-10T06:16:33,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742342_1518 (size=13153) 2024-12-10T06:16:33,342 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/46f9b31a1486440b9f08bdbf52e3ebbc as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/46f9b31a1486440b9f08bdbf52e3ebbc 2024-12-10T06:16:33,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/6386301e9cd042c3995f9dba2357e919 is 50, key is test_row_0/B:col10/1733811392608/Put/seqid=0 2024-12-10T06:16:33,346 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/B of 7438e40c5c6946ab37381c5eba2a38a0 into 46f9b31a1486440b9f08bdbf52e3ebbc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:33,346 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:33,346 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/B, priority=13, startTime=1733811393321; duration=0sec 2024-12-10T06:16:33,346 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:33,346 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:B 2024-12-10T06:16:33,346 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:33,347 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:33,347 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 7438e40c5c6946ab37381c5eba2a38a0/C is initiating minor compaction (all files) 2024-12-10T06:16:33,347 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7438e40c5c6946ab37381c5eba2a38a0/C in TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,347 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/66a0850cbcda4c8194311425f747fe54, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/741e416df71c4994875225bf52f42faf, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1bdcf2145234443c8ca8277cf718021c] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp, totalSize=36.8 K 2024-12-10T06:16:33,348 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a0850cbcda4c8194311425f747fe54, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733811388280 2024-12-10T06:16:33,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742343_1519 (size=12301) 2024-12-10T06:16:33,349 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 741e416df71c4994875225bf52f42faf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733811390632 2024-12-10T06:16:33,349 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bdcf2145234443c8ca8277cf718021c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733811391871 2024-12-10T06:16:33,356 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7438e40c5c6946ab37381c5eba2a38a0#C#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:33,357 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/3861e81015214bef8e520a64bf8c2176 is 50, key is test_row_0/C:col10/1733811391884/Put/seqid=0 2024-12-10T06:16:33,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742344_1520 (size=13153) 2024-12-10T06:16:33,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811453371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:33,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811453371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:33,373 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/3861e81015214bef8e520a64bf8c2176 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/3861e81015214bef8e520a64bf8c2176 2024-12-10T06:16:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:33,378 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/C of 7438e40c5c6946ab37381c5eba2a38a0 into 3861e81015214bef8e520a64bf8c2176(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:33,378 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:33,378 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/C, priority=13, startTime=1733811393321; duration=0sec 2024-12-10T06:16:33,378 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:33,378 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:C 2024-12-10T06:16:33,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:33,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:33,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811453486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811453486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:33,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:33,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T06:16:33,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811453706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:33,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811453706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:33,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,741 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/1b77102fac094dc693632e2b49a782b7 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1b77102fac094dc693632e2b49a782b7 2024-12-10T06:16:33,745 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7438e40c5c6946ab37381c5eba2a38a0/A of 7438e40c5c6946ab37381c5eba2a38a0 into 1b77102fac094dc693632e2b49a782b7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:33,745 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:33,745 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., storeName=7438e40c5c6946ab37381c5eba2a38a0/A, priority=13, startTime=1733811393321; duration=0sec 2024-12-10T06:16:33,745 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:33,745 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7438e40c5c6946ab37381c5eba2a38a0:A 2024-12-10T06:16:33,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/6386301e9cd042c3995f9dba2357e919 2024-12-10T06:16:33,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/5abaf3aa35894d3386dd303806bed411 is 50, key is test_row_0/C:col10/1733811392608/Put/seqid=0 2024-12-10T06:16:33,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742345_1521 (size=12301) 2024-12-10T06:16:33,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47676 deadline: 1733811453826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:33,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:33,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:33,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:33,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:34,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47658 deadline: 1733811454025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:34,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47696 deadline: 1733811454025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 , the old value is region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., hostname=c42c682b34f7,43983,1733811247584, seqNum=2 because the exception is null or not the one we care about 2024-12-10T06:16:34,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:34,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:34,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:34,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:34,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:34,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:34,112 INFO [Thread-1990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1990. 2024-12-10T06:16:34,112 DEBUG [Thread-1990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:34,112 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,112 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:34,121 INFO [Thread-1994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1994. 2024-12-10T06:16:34,121 DEBUG [Thread-1994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:34,121 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,121 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:34,123 INFO [Thread-1992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1992. 2024-12-10T06:16:34,123 DEBUG [Thread-1992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:34,123 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,123 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:34,126 INFO [Thread-1996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1996. 2024-12-10T06:16:34,126 DEBUG [Thread-1996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:34,126 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,127 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:34,131 INFO [Thread-1988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1988. 2024-12-10T06:16:34,131 DEBUG [Thread-1988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:34,131 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,131 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:34,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/5abaf3aa35894d3386dd303806bed411 2024-12-10T06:16:34,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/4913e443e605461ca9b9c6b440a6410c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/4913e443e605461ca9b9c6b440a6410c 2024-12-10T06:16:34,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/4913e443e605461ca9b9c6b440a6410c, entries=250, sequenceid=380, filesize=16.8 K 2024-12-10T06:16:34,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/6386301e9cd042c3995f9dba2357e919 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6386301e9cd042c3995f9dba2357e919 2024-12-10T06:16:34,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6386301e9cd042c3995f9dba2357e919, entries=150, sequenceid=380, filesize=12.0 K 2024-12-10T06:16:34,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/5abaf3aa35894d3386dd303806bed411 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5abaf3aa35894d3386dd303806bed411 2024-12-10T06:16:34,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5abaf3aa35894d3386dd303806bed411, entries=150, sequenceid=380, filesize=12.0 K 2024-12-10T06:16:34,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 7438e40c5c6946ab37381c5eba2a38a0 in 849ms, sequenceid=380, compaction requested=false 2024-12-10T06:16:34,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:34,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T06:16:34,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:34,191 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T06:16:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:34,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/081a921b8d5c47b9b4153d44e8b6d780 is 50, key is test_row_0/A:col10/1733811393369/Put/seqid=0 2024-12-10T06:16:34,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742346_1522 (size=12301) 2024-12-10T06:16:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:34,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. as already flushing 2024-12-10T06:16:34,547 INFO [Thread-1979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1979. 2024-12-10T06:16:34,547 INFO [Thread-1981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1981. 2024-12-10T06:16:34,547 DEBUG [Thread-1979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:34,547 DEBUG [Thread-1981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:34,547 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,547 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,547 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,547 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,547 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:34,547 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:34,598 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/081a921b8d5c47b9b4153d44e8b6d780 2024-12-10T06:16:34,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/db7fa89917334bcba11f27bc6fef2e75 is 50, key is test_row_0/B:col10/1733811393369/Put/seqid=0 2024-12-10T06:16:34,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742347_1523 (size=12301) 2024-12-10T06:16:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T06:16:34,847 INFO [Thread-1977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1977. 2024-12-10T06:16:34,847 DEBUG [Thread-1977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:34,847 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,847 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:34,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:35,007 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/db7fa89917334bcba11f27bc6fef2e75 2024-12-10T06:16:35,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/25b58f61b28c442e89656a91c6167b3e is 50, key is test_row_0/C:col10/1733811393369/Put/seqid=0 2024-12-10T06:16:35,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742348_1524 (size=12301) 2024-12-10T06:16:35,417 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/25b58f61b28c442e89656a91c6167b3e 2024-12-10T06:16:35,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/081a921b8d5c47b9b4153d44e8b6d780 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/081a921b8d5c47b9b4153d44e8b6d780 2024-12-10T06:16:35,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/081a921b8d5c47b9b4153d44e8b6d780, entries=150, sequenceid=392, filesize=12.0 K 2024-12-10T06:16:35,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/db7fa89917334bcba11f27bc6fef2e75 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/db7fa89917334bcba11f27bc6fef2e75 2024-12-10T06:16:35,426 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/db7fa89917334bcba11f27bc6fef2e75, entries=150, sequenceid=392, filesize=12.0 K 2024-12-10T06:16:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/25b58f61b28c442e89656a91c6167b3e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/25b58f61b28c442e89656a91c6167b3e 2024-12-10T06:16:35,430 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/25b58f61b28c442e89656a91c6167b3e, entries=150, sequenceid=392, filesize=12.0 K 2024-12-10T06:16:35,430 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=20.13 KB/20610 for 7438e40c5c6946ab37381c5eba2a38a0 in 1239ms, sequenceid=392, compaction requested=true 2024-12-10T06:16:35,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 7438e40c5c6946ab37381c5eba2a38a0: 2024-12-10T06:16:35,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:35,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-10T06:16:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-10T06:16:35,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-10T06:16:35,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9240 sec 2024-12-10T06:16:35,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.9280 sec 2024-12-10T06:16:35,916 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T06:16:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T06:16:36,675 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:37,937 INFO [Thread-1983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1983. 2024-12-10T06:16:37,937 DEBUG [Thread-1983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:37,937 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:37,937 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:37,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:38,027 INFO [Thread-1985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1985. 2024-12-10T06:16:38,027 DEBUG [Thread-1985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:16:38,027 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:38,027 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:38,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1065 2024-12-10T06:16:38,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3190 rows 2024-12-10T06:16:38,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1061 2024-12-10T06:16:38,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3179 rows 2024-12-10T06:16:38,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1065 2024-12-10T06:16:38,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3190 rows 2024-12-10T06:16:38,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1061 2024-12-10T06:16:38,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3180 rows 2024-12-10T06:16:38,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1055 2024-12-10T06:16:38,028 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3165 rows 2024-12-10T06:16:38,028 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:16:38,028 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:16:38,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:38,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:38,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:16:38,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T06:16:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T06:16:38,031 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811398031"}]},"ts":"1733811398031"} 2024-12-10T06:16:38,032 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T06:16:38,032 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T06:16:38,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T06:16:38,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7438e40c5c6946ab37381c5eba2a38a0, UNASSIGN}] 2024-12-10T06:16:38,034 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7438e40c5c6946ab37381c5eba2a38a0, UNASSIGN 2024-12-10T06:16:38,035 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=7438e40c5c6946ab37381c5eba2a38a0, regionState=CLOSING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:38,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7438e40c5c6946ab37381c5eba2a38a0, UNASSIGN because future has completed 2024-12-10T06:16:38,036 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:16:38,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:16:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T06:16:38,188 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 7438e40c5c6946ab37381c5eba2a38a0, disabling compactions & flushes 2024-12-10T06:16:38,189 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. after waiting 0 ms 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:38,189 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 7438e40c5c6946ab37381c5eba2a38a0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=A 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=B 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7438e40c5c6946ab37381c5eba2a38a0, store=C 2024-12-10T06:16:38,189 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:38,192 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/196dbdc91942469db85ab2c40522d80f is 50, key is test_row_0/A:col10/1733811397936/Put/seqid=0 2024-12-10T06:16:38,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742349_1525 (size=12301) 2024-12-10T06:16:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T06:16:38,596 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/196dbdc91942469db85ab2c40522d80f 2024-12-10T06:16:38,602 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/603bc5de58874223ace9a102ade0597f is 50, key is test_row_0/B:col10/1733811397936/Put/seqid=0 2024-12-10T06:16:38,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742350_1526 (size=12301) 2024-12-10T06:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T06:16:39,005 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/603bc5de58874223ace9a102ade0597f 2024-12-10T06:16:39,011 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/1f614b7bcef244d2b73d70b98f94a5f9 is 50, key is test_row_0/C:col10/1733811397936/Put/seqid=0 2024-12-10T06:16:39,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742351_1527 (size=12301) 2024-12-10T06:16:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T06:16:39,414 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/1f614b7bcef244d2b73d70b98f94a5f9 2024-12-10T06:16:39,418 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/A/196dbdc91942469db85ab2c40522d80f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/196dbdc91942469db85ab2c40522d80f 2024-12-10T06:16:39,420 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/196dbdc91942469db85ab2c40522d80f, entries=150, sequenceid=400, filesize=12.0 K 2024-12-10T06:16:39,421 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/B/603bc5de58874223ace9a102ade0597f as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/603bc5de58874223ace9a102ade0597f 2024-12-10T06:16:39,424 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/603bc5de58874223ace9a102ade0597f, entries=150, sequenceid=400, filesize=12.0 K 2024-12-10T06:16:39,424 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/.tmp/C/1f614b7bcef244d2b73d70b98f94a5f9 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1f614b7bcef244d2b73d70b98f94a5f9 2024-12-10T06:16:39,427 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1f614b7bcef244d2b73d70b98f94a5f9, entries=150, sequenceid=400, filesize=12.0 K 2024-12-10T06:16:39,428 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7438e40c5c6946ab37381c5eba2a38a0 in 1239ms, sequenceid=400, compaction requested=true 2024-12-10T06:16:39,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/0cd82860a5d64d189a6978649200c527, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/badfd0729b984aa09f399e14805f52fb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ac5cfe22bcef420aa3aff52c8b6f843c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/74a91ea8b60c488e9075e9097349f279, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/50b28f43b736469fbb8660f88a20f5d3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/f15a6bdb955b43e2a6215efb6394754a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/8fc941b7ba94402da2b748c2a1e61296, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ec5cf0686bd14b05ba8617b63ce1997a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49674dd6a7154e3ab94f24f3376cd4dd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/6a822d17daaa45409228c48dfd02f940, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49c2a66593454dfea84c99cc15e941c9, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/7c89c3044aa645bf96965eba2cfa8601, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/faadc946fdca4d0da3e7ef9608022142, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1b1029c654054ec3aba25c76f3bbf15c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b5865dbcc1a144409e6fa2933947fbcb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/06585a27be434116ac855b6296e55abd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b20caf03d8fe4cb093bd0bedce71c1db, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/c01618f74a134a75a1e942d96cf1053f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/22505db64af04612b332442d68df4a31, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1c53e0c6807b4866b5ad6c3e9a8acea4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a708ccace9424b1da139c5adb443e451, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a49106af8686462fa32245c8840a6c00, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/953d337407d04d93abef92f746625e1a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a931abdb9ac449ff8224436ea458d7c7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/bbc56781a7ba4e63aa15a0104e8b5a88] to archive 2024-12-10T06:16:39,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:16:39,431 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/0cd82860a5d64d189a6978649200c527 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/0cd82860a5d64d189a6978649200c527 2024-12-10T06:16:39,431 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ac5cfe22bcef420aa3aff52c8b6f843c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ac5cfe22bcef420aa3aff52c8b6f843c 2024-12-10T06:16:39,431 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/f15a6bdb955b43e2a6215efb6394754a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/f15a6bdb955b43e2a6215efb6394754a 2024-12-10T06:16:39,431 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/50b28f43b736469fbb8660f88a20f5d3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/50b28f43b736469fbb8660f88a20f5d3 2024-12-10T06:16:39,431 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/74a91ea8b60c488e9075e9097349f279 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/74a91ea8b60c488e9075e9097349f279 2024-12-10T06:16:39,431 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/8fc941b7ba94402da2b748c2a1e61296 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/8fc941b7ba94402da2b748c2a1e61296 2024-12-10T06:16:39,431 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ec5cf0686bd14b05ba8617b63ce1997a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/ec5cf0686bd14b05ba8617b63ce1997a 2024-12-10T06:16:39,432 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/badfd0729b984aa09f399e14805f52fb to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/badfd0729b984aa09f399e14805f52fb 2024-12-10T06:16:39,433 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49674dd6a7154e3ab94f24f3376cd4dd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49674dd6a7154e3ab94f24f3376cd4dd 2024-12-10T06:16:39,433 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/6a822d17daaa45409228c48dfd02f940 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/6a822d17daaa45409228c48dfd02f940 2024-12-10T06:16:39,433 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49c2a66593454dfea84c99cc15e941c9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/49c2a66593454dfea84c99cc15e941c9 2024-12-10T06:16:39,433 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/faadc946fdca4d0da3e7ef9608022142 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/faadc946fdca4d0da3e7ef9608022142 2024-12-10T06:16:39,433 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/7c89c3044aa645bf96965eba2cfa8601 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/7c89c3044aa645bf96965eba2cfa8601 2024-12-10T06:16:39,433 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1b1029c654054ec3aba25c76f3bbf15c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1b1029c654054ec3aba25c76f3bbf15c 2024-12-10T06:16:39,433 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b5865dbcc1a144409e6fa2933947fbcb to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b5865dbcc1a144409e6fa2933947fbcb 2024-12-10T06:16:39,434 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/06585a27be434116ac855b6296e55abd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/06585a27be434116ac855b6296e55abd 2024-12-10T06:16:39,435 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b20caf03d8fe4cb093bd0bedce71c1db to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/b20caf03d8fe4cb093bd0bedce71c1db 2024-12-10T06:16:39,435 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/c01618f74a134a75a1e942d96cf1053f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/c01618f74a134a75a1e942d96cf1053f 2024-12-10T06:16:39,435 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/953d337407d04d93abef92f746625e1a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/953d337407d04d93abef92f746625e1a 2024-12-10T06:16:39,435 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/22505db64af04612b332442d68df4a31 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/22505db64af04612b332442d68df4a31 2024-12-10T06:16:39,435 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a49106af8686462fa32245c8840a6c00 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a49106af8686462fa32245c8840a6c00 2024-12-10T06:16:39,435 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1c53e0c6807b4866b5ad6c3e9a8acea4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1c53e0c6807b4866b5ad6c3e9a8acea4 2024-12-10T06:16:39,435 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a708ccace9424b1da139c5adb443e451 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a708ccace9424b1da139c5adb443e451 2024-12-10T06:16:39,435 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a931abdb9ac449ff8224436ea458d7c7 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/a931abdb9ac449ff8224436ea458d7c7 2024-12-10T06:16:39,436 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/bbc56781a7ba4e63aa15a0104e8b5a88 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/bbc56781a7ba4e63aa15a0104e8b5a88 2024-12-10T06:16:39,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/33b71edb75a84f429c9889bd964edac1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9da9b6ad15cb4c74977bbd06f5b6b768, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2938d309b8584b8fa874e13e9ce4f7bd, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/61fdbbc101f3430baea49ca810c0664b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8ca7a0c2f6de4c38b2c2fb8770119338, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/aafc8c7796bd45038bf29e2ae031fade, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c437a76ea82b419abc51dd08d31b0c80, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/e8883884e4674afe95782b2d49310da3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/f178a28d0606497a978ee21f83314838, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/67f4418300b8489cbc650361f47c7a9e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6ef8a4b9cb214bddb6ff69d0a9df9a48, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2457134b7c2b424fb3fc838fa59e0f3e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/ee73cad0f834403e97f09c7907bfeaa3, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8d372fd224ac46a4a4034b5f9b361ebb, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6d57aa43d4ef4c0ca8efec6749c2f382, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/a9d82a42cf8546389914ffc0eda8cc13, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2aef918b0a0e4027abead54fd267ea4f, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c9df76e4b9774065b7615d7fa0cf99b6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/fab6ea83c0f84ab680ba6ed193003586, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/83443bd2114e489d84c3456ed586906c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c2ca7348f57748abb9703f72a976c20c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9e670c82e8e14e3ca71aa8012ce34f7d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/68eda72f46fc4d0ebc86fa24068aa376, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/3a44695ad502441596a00c348bb8b04d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/182bd38e4c74417294a6893fc8533fa2] to archive 2024-12-10T06:16:39,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:16:39,439 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2938d309b8584b8fa874e13e9ce4f7bd to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2938d309b8584b8fa874e13e9ce4f7bd 2024-12-10T06:16:39,439 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/33b71edb75a84f429c9889bd964edac1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/33b71edb75a84f429c9889bd964edac1 2024-12-10T06:16:39,440 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/e8883884e4674afe95782b2d49310da3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/e8883884e4674afe95782b2d49310da3 2024-12-10T06:16:39,440 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9da9b6ad15cb4c74977bbd06f5b6b768 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9da9b6ad15cb4c74977bbd06f5b6b768 2024-12-10T06:16:39,440 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/aafc8c7796bd45038bf29e2ae031fade to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/aafc8c7796bd45038bf29e2ae031fade 2024-12-10T06:16:39,440 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8ca7a0c2f6de4c38b2c2fb8770119338 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8ca7a0c2f6de4c38b2c2fb8770119338 2024-12-10T06:16:39,440 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/61fdbbc101f3430baea49ca810c0664b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/61fdbbc101f3430baea49ca810c0664b 2024-12-10T06:16:39,440 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c437a76ea82b419abc51dd08d31b0c80 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c437a76ea82b419abc51dd08d31b0c80 2024-12-10T06:16:39,441 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/f178a28d0606497a978ee21f83314838 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/f178a28d0606497a978ee21f83314838 2024-12-10T06:16:39,441 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2457134b7c2b424fb3fc838fa59e0f3e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2457134b7c2b424fb3fc838fa59e0f3e 2024-12-10T06:16:39,441 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6ef8a4b9cb214bddb6ff69d0a9df9a48 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6ef8a4b9cb214bddb6ff69d0a9df9a48 2024-12-10T06:16:39,441 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8d372fd224ac46a4a4034b5f9b361ebb to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/8d372fd224ac46a4a4034b5f9b361ebb 2024-12-10T06:16:39,441 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/67f4418300b8489cbc650361f47c7a9e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/67f4418300b8489cbc650361f47c7a9e 2024-12-10T06:16:39,441 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/ee73cad0f834403e97f09c7907bfeaa3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/ee73cad0f834403e97f09c7907bfeaa3 2024-12-10T06:16:39,442 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6d57aa43d4ef4c0ca8efec6749c2f382 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6d57aa43d4ef4c0ca8efec6749c2f382 2024-12-10T06:16:39,442 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/a9d82a42cf8546389914ffc0eda8cc13 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/a9d82a42cf8546389914ffc0eda8cc13 2024-12-10T06:16:39,443 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2aef918b0a0e4027abead54fd267ea4f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/2aef918b0a0e4027abead54fd267ea4f 2024-12-10T06:16:39,443 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c9df76e4b9774065b7615d7fa0cf99b6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c9df76e4b9774065b7615d7fa0cf99b6 2024-12-10T06:16:39,443 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9e670c82e8e14e3ca71aa8012ce34f7d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/9e670c82e8e14e3ca71aa8012ce34f7d 2024-12-10T06:16:39,443 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/83443bd2114e489d84c3456ed586906c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/83443bd2114e489d84c3456ed586906c 2024-12-10T06:16:39,443 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/fab6ea83c0f84ab680ba6ed193003586 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/fab6ea83c0f84ab680ba6ed193003586 2024-12-10T06:16:39,443 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c2ca7348f57748abb9703f72a976c20c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/c2ca7348f57748abb9703f72a976c20c 2024-12-10T06:16:39,443 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/68eda72f46fc4d0ebc86fa24068aa376 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/68eda72f46fc4d0ebc86fa24068aa376 2024-12-10T06:16:39,444 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/3a44695ad502441596a00c348bb8b04d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/3a44695ad502441596a00c348bb8b04d 2024-12-10T06:16:39,444 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/182bd38e4c74417294a6893fc8533fa2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/182bd38e4c74417294a6893fc8533fa2 2024-12-10T06:16:39,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/733647599c5c47d988e130b5683a57bc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4d82ac91924240b586a706ec76a61820, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/09ee7fc74d9b44e88490c7748c7c9eae, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/3e69841077804e658aae0eba62ed83b4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7ee6457b3f224814a5d53ef76049d6fa, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/de9b6b27bf254439a176d1c70ba4991b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/ae0a68e003034dbfb807a4fb74b162b0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/574c4bfa98164553b0e4e6fb30b7a99c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/808ddbe6854241ac8acce439a74dc19e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/0d5387196bd14c85a2352d927fe16eb2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/cfa1f7df17d74414a0e1d7ad919e50f8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/b74281b9edd64ab6871cf61758b926a2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/164935b4aaec49ffa640f13627d101f1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7172ed29d4834440bd7d398de548f808, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5b33e7e8d37e40b4bd7f25a86cd0eeaa, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/75926a4f5db74e5692f2d9483ad52be0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/88ecf4a262a04b60993f2fd1ce6a5558, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4f5b82f3b12f433aba570e203ffad92e, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/54902d707e9c479da11914753a01578b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/159e0b016d294149ad7a81dd7a3e967b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/692595494d5b425f820b1f268d7d05ba, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/66a0850cbcda4c8194311425f747fe54, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/9cfbbef3e7ec49d3a1fe201d4918d048, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/741e416df71c4994875225bf52f42faf, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1bdcf2145234443c8ca8277cf718021c] to archive 2024-12-10T06:16:39,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:16:39,447 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4d82ac91924240b586a706ec76a61820 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4d82ac91924240b586a706ec76a61820 2024-12-10T06:16:39,447 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/ae0a68e003034dbfb807a4fb74b162b0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/ae0a68e003034dbfb807a4fb74b162b0 2024-12-10T06:16:39,447 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/733647599c5c47d988e130b5683a57bc to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/733647599c5c47d988e130b5683a57bc 2024-12-10T06:16:39,448 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/09ee7fc74d9b44e88490c7748c7c9eae to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/09ee7fc74d9b44e88490c7748c7c9eae 2024-12-10T06:16:39,448 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7ee6457b3f224814a5d53ef76049d6fa to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7ee6457b3f224814a5d53ef76049d6fa 2024-12-10T06:16:39,448 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/de9b6b27bf254439a176d1c70ba4991b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/de9b6b27bf254439a176d1c70ba4991b 2024-12-10T06:16:39,448 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/574c4bfa98164553b0e4e6fb30b7a99c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/574c4bfa98164553b0e4e6fb30b7a99c 2024-12-10T06:16:39,448 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/3e69841077804e658aae0eba62ed83b4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/3e69841077804e658aae0eba62ed83b4 2024-12-10T06:16:39,449 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/808ddbe6854241ac8acce439a74dc19e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/808ddbe6854241ac8acce439a74dc19e 2024-12-10T06:16:39,449 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/cfa1f7df17d74414a0e1d7ad919e50f8 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/cfa1f7df17d74414a0e1d7ad919e50f8 2024-12-10T06:16:39,449 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/b74281b9edd64ab6871cf61758b926a2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/b74281b9edd64ab6871cf61758b926a2 2024-12-10T06:16:39,449 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/164935b4aaec49ffa640f13627d101f1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/164935b4aaec49ffa640f13627d101f1 2024-12-10T06:16:39,449 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/0d5387196bd14c85a2352d927fe16eb2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/0d5387196bd14c85a2352d927fe16eb2 2024-12-10T06:16:39,450 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7172ed29d4834440bd7d398de548f808 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/7172ed29d4834440bd7d398de548f808 2024-12-10T06:16:39,450 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5b33e7e8d37e40b4bd7f25a86cd0eeaa to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5b33e7e8d37e40b4bd7f25a86cd0eeaa 2024-12-10T06:16:39,450 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/75926a4f5db74e5692f2d9483ad52be0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/75926a4f5db74e5692f2d9483ad52be0 2024-12-10T06:16:39,451 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4f5b82f3b12f433aba570e203ffad92e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/4f5b82f3b12f433aba570e203ffad92e 2024-12-10T06:16:39,451 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/54902d707e9c479da11914753a01578b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/54902d707e9c479da11914753a01578b 2024-12-10T06:16:39,451 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/88ecf4a262a04b60993f2fd1ce6a5558 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/88ecf4a262a04b60993f2fd1ce6a5558 2024-12-10T06:16:39,451 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/692595494d5b425f820b1f268d7d05ba to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/692595494d5b425f820b1f268d7d05ba 2024-12-10T06:16:39,451 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/159e0b016d294149ad7a81dd7a3e967b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/159e0b016d294149ad7a81dd7a3e967b 2024-12-10T06:16:39,451 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/66a0850cbcda4c8194311425f747fe54 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/66a0850cbcda4c8194311425f747fe54 2024-12-10T06:16:39,452 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/9cfbbef3e7ec49d3a1fe201d4918d048 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/9cfbbef3e7ec49d3a1fe201d4918d048 2024-12-10T06:16:39,452 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/741e416df71c4994875225bf52f42faf to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/741e416df71c4994875225bf52f42faf 2024-12-10T06:16:39,452 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1bdcf2145234443c8ca8277cf718021c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1bdcf2145234443c8ca8277cf718021c 2024-12-10T06:16:39,456 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/recovered.edits/403.seqid, newMaxSeqId=403, maxSeqId=1 2024-12-10T06:16:39,457 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. 2024-12-10T06:16:39,457 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 7438e40c5c6946ab37381c5eba2a38a0: Waiting for close lock at 1733811398189Running coprocessor pre-close hooks at 1733811398189Disabling compacts and flushes for region at 1733811398189Disabling writes for close at 1733811398189Obtaining lock to block concurrent updates at 1733811398189Preparing flush snapshotting stores in 7438e40c5c6946ab37381c5eba2a38a0 at 1733811398189Finished memstore snapshotting TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733811398189Flushing stores of TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0. at 1733811398190 (+1 ms)Flushing 7438e40c5c6946ab37381c5eba2a38a0/A: creating writer at 1733811398190Flushing 7438e40c5c6946ab37381c5eba2a38a0/A: appending metadata at 1733811398192 (+2 ms)Flushing 7438e40c5c6946ab37381c5eba2a38a0/A: closing flushed file at 1733811398192Flushing 7438e40c5c6946ab37381c5eba2a38a0/B: creating writer at 1733811398600 (+408 ms)Flushing 7438e40c5c6946ab37381c5eba2a38a0/B: appending metadata at 1733811398602 (+2 ms)Flushing 7438e40c5c6946ab37381c5eba2a38a0/B: closing flushed file at 1733811398602Flushing 7438e40c5c6946ab37381c5eba2a38a0/C: creating writer at 1733811399008 (+406 ms)Flushing 7438e40c5c6946ab37381c5eba2a38a0/C: appending metadata at 1733811399010 (+2 ms)Flushing 7438e40c5c6946ab37381c5eba2a38a0/C: closing flushed file at 1733811399011 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@134b54c5: reopening flushed file at 1733811399417 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2992e9bb: reopening flushed file at 1733811399421 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e52b940: reopening flushed file at 1733811399424 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7438e40c5c6946ab37381c5eba2a38a0 in 1239ms, sequenceid=400, compaction requested=true at 1733811399428 (+4 ms)Writing region close event to WAL at 1733811399454 (+26 ms)Running coprocessor post-close hooks at 1733811399457 (+3 ms)Closed at 1733811399457 2024-12-10T06:16:39,459 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:39,459 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=7438e40c5c6946ab37381c5eba2a38a0, regionState=CLOSED 2024-12-10T06:16:39,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:16:39,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-10T06:16:39,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 7438e40c5c6946ab37381c5eba2a38a0, server=c42c682b34f7,43983,1733811247584 in 1.4260 sec 2024-12-10T06:16:39,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-12-10T06:16:39,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7438e40c5c6946ab37381c5eba2a38a0, UNASSIGN in 1.4290 sec 2024-12-10T06:16:39,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-10T06:16:39,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4320 sec 2024-12-10T06:16:39,468 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811399467"}]},"ts":"1733811399467"} 2024-12-10T06:16:39,469 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T06:16:39,469 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T06:16:39,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4400 sec 2024-12-10T06:16:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T06:16:40,175 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:40,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T06:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:40,178 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-10T06:16:40,178 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:40,180 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:40,181 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/recovered.edits] 2024-12-10T06:16:40,183 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/196dbdc91942469db85ab2c40522d80f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/196dbdc91942469db85ab2c40522d80f 2024-12-10T06:16:40,183 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/081a921b8d5c47b9b4153d44e8b6d780 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/081a921b8d5c47b9b4153d44e8b6d780 2024-12-10T06:16:40,184 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/4913e443e605461ca9b9c6b440a6410c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/4913e443e605461ca9b9c6b440a6410c 2024-12-10T06:16:40,184 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1b77102fac094dc693632e2b49a782b7 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/A/1b77102fac094dc693632e2b49a782b7 2024-12-10T06:16:40,186 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/46f9b31a1486440b9f08bdbf52e3ebbc to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/46f9b31a1486440b9f08bdbf52e3ebbc 2024-12-10T06:16:40,186 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/db7fa89917334bcba11f27bc6fef2e75 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/db7fa89917334bcba11f27bc6fef2e75 2024-12-10T06:16:40,186 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/603bc5de58874223ace9a102ade0597f to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/603bc5de58874223ace9a102ade0597f 2024-12-10T06:16:40,186 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6386301e9cd042c3995f9dba2357e919 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/B/6386301e9cd042c3995f9dba2357e919 2024-12-10T06:16:40,188 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/25b58f61b28c442e89656a91c6167b3e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/25b58f61b28c442e89656a91c6167b3e 2024-12-10T06:16:40,188 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1f614b7bcef244d2b73d70b98f94a5f9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/1f614b7bcef244d2b73d70b98f94a5f9 2024-12-10T06:16:40,188 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/3861e81015214bef8e520a64bf8c2176 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/3861e81015214bef8e520a64bf8c2176 2024-12-10T06:16:40,188 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5abaf3aa35894d3386dd303806bed411 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/C/5abaf3aa35894d3386dd303806bed411 2024-12-10T06:16:40,190 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/recovered.edits/403.seqid to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0/recovered.edits/403.seqid 2024-12-10T06:16:40,191 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/7438e40c5c6946ab37381c5eba2a38a0 2024-12-10T06:16:40,191 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T06:16:40,192 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:40,194 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T06:16:40,196 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T06:16:40,196 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:40,196 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T06:16:40,197 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733811400196"}]},"ts":"9223372036854775807"} 2024-12-10T06:16:40,198 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T06:16:40,198 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7438e40c5c6946ab37381c5eba2a38a0, NAME => 'TestAcidGuarantees,,1733811371924.7438e40c5c6946ab37381c5eba2a38a0.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T06:16:40,198 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T06:16:40,198 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733811400198"}]},"ts":"9223372036854775807"} 2024-12-10T06:16:40,199 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T06:16:40,200 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:40,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 24 msec 2024-12-10T06:16:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-10T06:16:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T06:16:40,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:40,295 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=253 (was 255), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=351 (was 389), ProcessCount=11 (was 11), AvailableMemoryMB=5958 (was 6031) 2024-12-10T06:16:40,303 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=253, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=351, ProcessCount=11, AvailableMemoryMB=5958 2024-12-10T06:16:40,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T06:16:40,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:40,306 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T06:16:40,307 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:40,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-12-10T06:16:40,307 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T06:16:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T06:16:40,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742352_1528 (size=1061) 2024-12-10T06:16:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T06:16:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T06:16:40,713 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6 2024-12-10T06:16:40,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742353_1529 (size=53) 2024-12-10T06:16:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T06:16:41,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:16:41,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 112cbd3eaf4337ea756788d925ebbcd9, disabling compactions & flushes 2024-12-10T06:16:41,119 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:41,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:41,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. after waiting 0 ms 2024-12-10T06:16:41,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:41,119 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:41,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 112cbd3eaf4337ea756788d925ebbcd9: Waiting for close lock at 1733811401119Disabling compacts and flushes for region at 1733811401119Disabling writes for close at 1733811401119Writing region close event to WAL at 1733811401119Closed at 1733811401119 2024-12-10T06:16:41,120 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T06:16:41,120 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733811401120"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733811401120"}]},"ts":"1733811401120"} 2024-12-10T06:16:41,122 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T06:16:41,122 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T06:16:41,122 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811401122"}]},"ts":"1733811401122"} 2024-12-10T06:16:41,123 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T06:16:41,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, ASSIGN}] 2024-12-10T06:16:41,125 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, ASSIGN 2024-12-10T06:16:41,125 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, ASSIGN; state=OFFLINE, location=c42c682b34f7,43983,1733811247584; forceNewPlan=false, retain=false 2024-12-10T06:16:41,276 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=112cbd3eaf4337ea756788d925ebbcd9, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:41,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, ASSIGN because future has completed 2024-12-10T06:16:41,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:16:41,432 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:41,432 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:16:41,432 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,432 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:16:41,432 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,432 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,433 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,434 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:16:41,435 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 112cbd3eaf4337ea756788d925ebbcd9 columnFamilyName A 2024-12-10T06:16:41,435 DEBUG [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:41,435 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(327): Store=112cbd3eaf4337ea756788d925ebbcd9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:16:41,435 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,436 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:16:41,436 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 112cbd3eaf4337ea756788d925ebbcd9 columnFamilyName B 2024-12-10T06:16:41,436 DEBUG [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:41,437 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(327): Store=112cbd3eaf4337ea756788d925ebbcd9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:16:41,437 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,437 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:16:41,437 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 112cbd3eaf4337ea756788d925ebbcd9 columnFamilyName C 2024-12-10T06:16:41,438 DEBUG [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:41,438 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(327): Store=112cbd3eaf4337ea756788d925ebbcd9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:16:41,438 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:41,438 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,438 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,439 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,439 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,440 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,440 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:16:41,441 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,442 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:16:41,442 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 112cbd3eaf4337ea756788d925ebbcd9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65333825, jitterRate=-0.02645014226436615}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:16:41,442 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:41,443 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 112cbd3eaf4337ea756788d925ebbcd9: Running coprocessor pre-open hook at 1733811401433Writing region info on filesystem at 1733811401433Initializing all the Stores at 1733811401433Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811401433Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811401433Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811401433Cleaning up temporary data from old regions at 1733811401440 (+7 ms)Running coprocessor post-open hooks at 1733811401442 (+2 ms)Region opened successfully at 1733811401443 (+1 ms) 2024-12-10T06:16:41,443 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., pid=138, masterSystemTime=1733811401429 2024-12-10T06:16:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T06:16:41,445 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:41,445 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:41,446 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=112cbd3eaf4337ea756788d925ebbcd9, regionState=OPEN, openSeqNum=2, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:41,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:16:41,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-10T06:16:41,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 in 169 msec 2024-12-10T06:16:41,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-10T06:16:41,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, ASSIGN in 325 msec 2024-12-10T06:16:41,451 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T06:16:41,451 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811401451"}]},"ts":"1733811401451"} 2024-12-10T06:16:41,452 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T06:16:41,453 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T06:16:41,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-12-10T06:16:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T06:16:42,455 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:42,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:42,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:42,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:42,457 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ade1a04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:42,459 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59517139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:42,459 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:42,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:42,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:42,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:42,462 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T06:16:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c42c682b34f7,37415,1733811246834 2024-12-10T06:16:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6edced91 2024-12-10T06:16:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T06:16:42,463 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T06:16:42,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T06:16:42,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:16:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T06:16:42,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742354_1530 (size=1097) 2024-12-10T06:16:42,875 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T06:16:42,875 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T06:16:42,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T06:16:42,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, REOPEN/MOVE}] 2024-12-10T06:16:42,879 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, REOPEN/MOVE 2024-12-10T06:16:42,880 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=112cbd3eaf4337ea756788d925ebbcd9, regionState=CLOSING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:42,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, REOPEN/MOVE because future has completed 2024-12-10T06:16:42,881 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:16:42,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:16:43,033 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,033 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:16:43,033 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 112cbd3eaf4337ea756788d925ebbcd9, disabling compactions & flushes 2024-12-10T06:16:43,033 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,033 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,033 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. after waiting 0 ms 2024-12-10T06:16:43,033 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,036 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T06:16:43,037 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,037 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 112cbd3eaf4337ea756788d925ebbcd9: Waiting for close lock at 1733811403033Running coprocessor pre-close hooks at 1733811403033Disabling compacts and flushes for region at 1733811403033Disabling writes for close at 1733811403033Writing region close event to WAL at 1733811403034 (+1 ms)Running coprocessor post-close hooks at 1733811403037 (+3 ms)Closed at 1733811403037 2024-12-10T06:16:43,037 WARN [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 112cbd3eaf4337ea756788d925ebbcd9 to self. 2024-12-10T06:16:43,038 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,039 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=112cbd3eaf4337ea756788d925ebbcd9, regionState=CLOSED 2024-12-10T06:16:43,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:16:43,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-10T06:16:43,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 in 160 msec 2024-12-10T06:16:43,042 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, REOPEN/MOVE; state=CLOSED, location=c42c682b34f7,43983,1733811247584; forceNewPlan=false, retain=true 2024-12-10T06:16:43,193 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=112cbd3eaf4337ea756788d925ebbcd9, regionState=OPENING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, REOPEN/MOVE because future has completed 2024-12-10T06:16:43,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:16:43,349 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,349 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:16:43,349 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,349 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:16:43,349 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,349 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,350 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,351 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:16:43,351 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 112cbd3eaf4337ea756788d925ebbcd9 columnFamilyName A 2024-12-10T06:16:43,352 DEBUG [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:43,352 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(327): Store=112cbd3eaf4337ea756788d925ebbcd9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:16:43,353 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,353 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:16:43,353 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 112cbd3eaf4337ea756788d925ebbcd9 columnFamilyName B 2024-12-10T06:16:43,353 DEBUG [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:43,354 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(327): Store=112cbd3eaf4337ea756788d925ebbcd9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:16:43,354 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,354 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T06:16:43,354 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 112cbd3eaf4337ea756788d925ebbcd9 columnFamilyName C 2024-12-10T06:16:43,354 DEBUG [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:43,354 INFO [StoreOpener-112cbd3eaf4337ea756788d925ebbcd9-1 {}] regionserver.HStore(327): Store=112cbd3eaf4337ea756788d925ebbcd9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:16:43,355 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,355 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,355 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,356 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,356 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,356 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,357 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:16:43,358 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,358 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 112cbd3eaf4337ea756788d925ebbcd9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64796885, jitterRate=-0.03445117175579071}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:16:43,358 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,359 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 112cbd3eaf4337ea756788d925ebbcd9: Running coprocessor pre-open hook at 1733811403349Writing region info on filesystem at 1733811403349Initializing all the Stores at 1733811403350 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811403350Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811403350Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733811403350Cleaning up temporary data from old regions at 1733811403356 (+6 ms)Running coprocessor post-open hooks at 1733811403358 (+2 ms)Region opened successfully at 1733811403359 (+1 ms) 2024-12-10T06:16:43,359 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., pid=143, masterSystemTime=1733811403346 2024-12-10T06:16:43,361 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,361 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,361 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=112cbd3eaf4337ea756788d925ebbcd9, regionState=OPEN, openSeqNum=5, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:16:43,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-12-10T06:16:43,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 in 168 msec 2024-12-10T06:16:43,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-10T06:16:43,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, REOPEN/MOVE in 486 msec 2024-12-10T06:16:43,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-10T06:16:43,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-12-10T06:16:43,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-12-10T06:16:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-10T06:16:43,370 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:43,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,372 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f4bbdef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,373 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3835b5ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,375 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3be2927e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,376 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1491fa62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,377 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56030d7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,378 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b37d2be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,380 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25aa2690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,381 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ed07df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,382 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,382 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ea70a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,383 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a92ed5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,384 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,385 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728fd2d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,386 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4142a804, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,387 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,387 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a743a2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,389 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f0b87e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,390 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dd966b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,391 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebdd4ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,392 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,393 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60787788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,394 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183ffc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@544c9525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,37415,-1 for getting cluster id 2024-12-10T06:16:43,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:16:43,395 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fa859e13-0c00-4715-a487-f5a4e3af8477' 2024-12-10T06:16:43,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:16:43,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fa859e13-0c00-4715-a487-f5a4e3af8477" 2024-12-10T06:16:43,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2432f172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,37415,-1] 2024-12-10T06:16:43,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:16:43,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:16:43,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:16:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a9c6c87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:16:43,397 DEBUG [Thread-2377 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,397 DEBUG [Thread-2380 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,397 DEBUG [Thread-2384 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,397 DEBUG [Thread-2373 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:43,397 DEBUG [Thread-2369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,397 DEBUG [Thread-2375 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,398 DEBUG [Thread-2382 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,398 DEBUG [Thread-2388 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,398 DEBUG [Thread-2371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-10T06:16:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,398 DEBUG [Thread-2386 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:16:43,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T06:16:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,401 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,401 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:43,401 DEBUG [Thread-2375 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,43983,1733811247584, seqNum=-1] 2024-12-10T06:16:43,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:43,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,402 DEBUG [Thread-2375 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:16:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:16:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5] 2024-12-10T06:16:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:16:43,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:43,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:43,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:43,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:43,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:43,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811463437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811463438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733811403435, max timestamp=1733811403441Num uniques 859; 2024-12-10T06:16:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:16:43,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733811403435, max timestamp=1733811403441Num uniques 859; 2024-12-10T06:16:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:16:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210349f8559ef8849d888ac6f54761519b2_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811403408/Put/seqid=0 2024-12-10T06:16:43,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733811403435, max timestamp=1733811403441Num uniques 859; 2024-12-10T06:16:43,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c42c682b34f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T06:16:43,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742355_1531 (size=12154) 2024-12-10T06:16:43,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811463474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811463474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811463481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T06:16:43,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:43,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:43,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:43,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:43,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811463555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811463555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811463586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811463586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811463595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:43,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:43,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:43,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T06:16:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811463766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811463766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811463795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811463795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811463805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:43,853 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:43,857 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210349f8559ef8849d888ac6f54761519b2_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210349f8559ef8849d888ac6f54761519b2_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:43,857 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/f9f182746e09421483fe3fc153516247, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:43,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/f9f182746e09421483fe3fc153516247 is 175, key is test_row_0/A:col10/1733811403408/Put/seqid=0 2024-12-10T06:16:43,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742356_1532 (size=30955) 2024-12-10T06:16:43,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:43,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T06:16:44,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811464075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811464075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811464105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811464105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811464116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,262 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/f9f182746e09421483fe3fc153516247 2024-12-10T06:16:44,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/ef27a7ede7a841ea8ed625a8c87e1758 is 50, key is test_row_0/B:col10/1733811403408/Put/seqid=0 2024-12-10T06:16:44,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742357_1533 (size=12001) 2024-12-10T06:16:44,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:44,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:44,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:44,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:44,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T06:16:44,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811464585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811464585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811464616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811464616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811464625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:44,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:44,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:44,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:44,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/ef27a7ede7a841ea8ed625a8c87e1758 2024-12-10T06:16:44,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/021acd5765384bb7960fd8b45ee15865 is 50, key is test_row_0/C:col10/1733811403408/Put/seqid=0 2024-12-10T06:16:44,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742358_1534 (size=12001) 2024-12-10T06:16:44,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:44,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:44,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:44,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:44,972 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:16:45,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:45,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:45,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:45,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:45,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:45,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:45,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/021acd5765384bb7960fd8b45ee15865 2024-12-10T06:16:45,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/f9f182746e09421483fe3fc153516247 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/f9f182746e09421483fe3fc153516247 2024-12-10T06:16:45,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/f9f182746e09421483fe3fc153516247, entries=150, sequenceid=14, filesize=30.2 K 2024-12-10T06:16:45,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/ef27a7ede7a841ea8ed625a8c87e1758 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ef27a7ede7a841ea8ed625a8c87e1758 2024-12-10T06:16:45,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ef27a7ede7a841ea8ed625a8c87e1758, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T06:16:45,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/021acd5765384bb7960fd8b45ee15865 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/021acd5765384bb7960fd8b45ee15865 2024-12-10T06:16:45,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/021acd5765384bb7960fd8b45ee15865, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T06:16:45,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 112cbd3eaf4337ea756788d925ebbcd9 in 1728ms, sequenceid=14, compaction requested=false 2024-12-10T06:16:45,139 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T06:16:45,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:45,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T06:16:45,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:45,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T06:16:45,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:45,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:16:45,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121063299451649148728c309683c17fef19_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811403472/Put/seqid=0 2024-12-10T06:16:45,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742359_1535 (size=12154) 2024-12-10T06:16:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T06:16:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:45,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:45,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811465624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811465624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:45,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811465625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:45,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811465625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:45,631 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 112cbd3eaf4337ea756788d925ebbcd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:45,631 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 112cbd3eaf4337ea756788d925ebbcd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:45,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811465645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:45,647 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 112cbd3eaf4337ea756788d925ebbcd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:45,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:45,659 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121063299451649148728c309683c17fef19_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121063299451649148728c309683c17fef19_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:45,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/36139701d7354c2fb9dc82d25b541bb0, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/36139701d7354c2fb9dc82d25b541bb0 is 175, key is test_row_0/A:col10/1733811403472/Put/seqid=0 2024-12-10T06:16:45,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742360_1536 (size=30955) 2024-12-10T06:16:45,665 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/36139701d7354c2fb9dc82d25b541bb0 2024-12-10T06:16:45,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8d30f8d3c370427da6920fe6df1e0967 is 50, key is test_row_0/B:col10/1733811403472/Put/seqid=0 2024-12-10T06:16:45,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742361_1537 (size=12001) 2024-12-10T06:16:45,677 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8d30f8d3c370427da6920fe6df1e0967 2024-12-10T06:16:45,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/4100529f0b14411c898818c24590913d is 50, key is test_row_0/C:col10/1733811403472/Put/seqid=0 2024-12-10T06:16:45,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742362_1538 (size=12001) 2024-12-10T06:16:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811465736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:45,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811465736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:45,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811465945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:45,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811465946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:46,088 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/4100529f0b14411c898818c24590913d 2024-12-10T06:16:46,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/36139701d7354c2fb9dc82d25b541bb0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/36139701d7354c2fb9dc82d25b541bb0 2024-12-10T06:16:46,095 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/36139701d7354c2fb9dc82d25b541bb0, entries=150, sequenceid=50, filesize=30.2 K 2024-12-10T06:16:46,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8d30f8d3c370427da6920fe6df1e0967 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8d30f8d3c370427da6920fe6df1e0967 2024-12-10T06:16:46,099 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8d30f8d3c370427da6920fe6df1e0967, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T06:16:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/4100529f0b14411c898818c24590913d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/4100529f0b14411c898818c24590913d 2024-12-10T06:16:46,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/4100529f0b14411c898818c24590913d, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T06:16:46,103 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 112cbd3eaf4337ea756788d925ebbcd9 in 860ms, sequenceid=50, compaction requested=false 2024-12-10T06:16:46,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:46,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:46,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-10T06:16:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-10T06:16:46,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-10T06:16:46,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7020 sec 2024-12-10T06:16:46,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.7090 sec 2024-12-10T06:16:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:46,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T06:16:46,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:46,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:46,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:46,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:46,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:46,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:46,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210729d17b190a647b0a7fa912a216b8ba7_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811406257/Put/seqid=0 2024-12-10T06:16:46,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742363_1539 (size=14594) 2024-12-10T06:16:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811466306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811466307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:46,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811466415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:46,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811466415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:46,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811466625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:46,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811466625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:46,677 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:46,680 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210729d17b190a647b0a7fa912a216b8ba7_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210729d17b190a647b0a7fa912a216b8ba7_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:46,681 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/2c7c9e0f409b4549b276edd82d143a22, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:46,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/2c7c9e0f409b4549b276edd82d143a22 is 175, key is test_row_0/A:col10/1733811406257/Put/seqid=0 2024-12-10T06:16:46,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742364_1540 (size=39549) 2024-12-10T06:16:46,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811466935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:46,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811466935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:47,086 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/2c7c9e0f409b4549b276edd82d143a22 2024-12-10T06:16:47,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/74dea80187094b868c099998af738d96 is 50, key is test_row_0/B:col10/1733811406257/Put/seqid=0 2024-12-10T06:16:47,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742365_1541 (size=12001) 2024-12-10T06:16:47,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811467445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811467445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:47,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/74dea80187094b868c099998af738d96 2024-12-10T06:16:47,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/51fad87d1f0041fd8188362eb9c30773 is 50, key is test_row_0/C:col10/1733811406257/Put/seqid=0 2024-12-10T06:16:47,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742366_1542 (size=12001) 2024-12-10T06:16:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T06:16:47,566 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:47,566 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-10T06:16:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T06:16:47,568 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:47,569 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:47,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:47,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811467645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:47,650 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 112cbd3eaf4337ea756788d925ebbcd9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:47,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811467655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:47,658 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 112cbd3eaf4337ea756788d925ebbcd9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:47,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811467665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:47,669 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 112cbd3eaf4337ea756788d925ebbcd9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T06:16:47,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-10T06:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:47,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:47,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-10T06:16:47,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:47,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:47,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:47,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:47,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T06:16:47,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/51fad87d1f0041fd8188362eb9c30773 2024-12-10T06:16:47,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/2c7c9e0f409b4549b276edd82d143a22 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/2c7c9e0f409b4549b276edd82d143a22 2024-12-10T06:16:47,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/2c7c9e0f409b4549b276edd82d143a22, entries=200, sequenceid=63, filesize=38.6 K 2024-12-10T06:16:47,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/74dea80187094b868c099998af738d96 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/74dea80187094b868c099998af738d96 2024-12-10T06:16:47,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/74dea80187094b868c099998af738d96, entries=150, sequenceid=63, filesize=11.7 K 2024-12-10T06:16:47,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/51fad87d1f0041fd8188362eb9c30773 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/51fad87d1f0041fd8188362eb9c30773 2024-12-10T06:16:47,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/51fad87d1f0041fd8188362eb9c30773, entries=150, sequenceid=63, filesize=11.7 K 2024-12-10T06:16:47,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 112cbd3eaf4337ea756788d925ebbcd9 in 1669ms, sequenceid=63, compaction requested=true 2024-12-10T06:16:47,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:47,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:47,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:47,927 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:47,927 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:47,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:47,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:47,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:47,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:47,928 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:47,928 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:47,928 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/B is initiating minor compaction (all files) 2024-12-10T06:16:47,928 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/A is initiating minor compaction (all files) 2024-12-10T06:16:47,928 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/B in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:47,928 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/A in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:47,928 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ef27a7ede7a841ea8ed625a8c87e1758, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8d30f8d3c370427da6920fe6df1e0967, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/74dea80187094b868c099998af738d96] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=35.2 K 2024-12-10T06:16:47,928 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/f9f182746e09421483fe3fc153516247, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/36139701d7354c2fb9dc82d25b541bb0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/2c7c9e0f409b4549b276edd82d143a22] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=99.1 K 2024-12-10T06:16:47,928 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:47,928 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/f9f182746e09421483fe3fc153516247, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/36139701d7354c2fb9dc82d25b541bb0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/2c7c9e0f409b4549b276edd82d143a22] 2024-12-10T06:16:47,928 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ef27a7ede7a841ea8ed625a8c87e1758, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733811403408 2024-12-10T06:16:47,928 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9f182746e09421483fe3fc153516247, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733811403408 2024-12-10T06:16:47,929 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d30f8d3c370427da6920fe6df1e0967, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733811403472 2024-12-10T06:16:47,929 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36139701d7354c2fb9dc82d25b541bb0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733811403472 2024-12-10T06:16:47,929 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 74dea80187094b868c099998af738d96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733811405608 2024-12-10T06:16:47,929 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c7c9e0f409b4549b276edd82d143a22, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733811405608 2024-12-10T06:16:47,936 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#B#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:47,936 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8c949bfec22b43d9bcd180949142de3d is 50, key is test_row_0/B:col10/1733811406257/Put/seqid=0 2024-12-10T06:16:47,937 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:47,939 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412101641512ee07b4bde8e544a7c333fea79_112cbd3eaf4337ea756788d925ebbcd9 store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:47,942 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412101641512ee07b4bde8e544a7c333fea79_112cbd3eaf4337ea756788d925ebbcd9, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:47,942 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101641512ee07b4bde8e544a7c333fea79_112cbd3eaf4337ea756788d925ebbcd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:47,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742368_1544 (size=4469) 2024-12-10T06:16:47,947 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#A#compaction#460 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:47,947 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/86544fe61f7d45fcb0864fd29140df71 is 175, key is test_row_0/A:col10/1733811406257/Put/seqid=0 2024-12-10T06:16:47,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742369_1545 (size=31058) 2024-12-10T06:16:47,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742367_1543 (size=12104) 2024-12-10T06:16:48,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-10T06:16:48,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:48,029 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T06:16:48,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:48,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:48,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:48,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:48,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:48,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:48,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a87c189696594b34b34d629f68cd8e2b_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811406305/Put/seqid=0 2024-12-10T06:16:48,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742370_1546 (size=12154) 2024-12-10T06:16:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T06:16:48,355 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/86544fe61f7d45fcb0864fd29140df71 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/86544fe61f7d45fcb0864fd29140df71 2024-12-10T06:16:48,359 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/A of 112cbd3eaf4337ea756788d925ebbcd9 into 86544fe61f7d45fcb0864fd29140df71(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:48,359 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:48,360 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/A, priority=13, startTime=1733811407927; duration=0sec 2024-12-10T06:16:48,360 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:48,360 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:A 2024-12-10T06:16:48,360 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:48,361 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:48,361 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/C is initiating minor compaction (all files) 2024-12-10T06:16:48,361 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/C in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:48,361 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/021acd5765384bb7960fd8b45ee15865, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/4100529f0b14411c898818c24590913d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/51fad87d1f0041fd8188362eb9c30773] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=35.2 K 2024-12-10T06:16:48,361 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8c949bfec22b43d9bcd180949142de3d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8c949bfec22b43d9bcd180949142de3d 2024-12-10T06:16:48,361 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 021acd5765384bb7960fd8b45ee15865, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733811403408 2024-12-10T06:16:48,362 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4100529f0b14411c898818c24590913d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733811403472 2024-12-10T06:16:48,362 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51fad87d1f0041fd8188362eb9c30773, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733811405608 2024-12-10T06:16:48,366 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/B of 112cbd3eaf4337ea756788d925ebbcd9 into 8c949bfec22b43d9bcd180949142de3d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:48,366 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:48,366 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/B, priority=13, startTime=1733811407927; duration=0sec 2024-12-10T06:16:48,366 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:48,366 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:B 2024-12-10T06:16:48,370 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:48,370 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/f55abdcd1de54d8b9d6e6e8cecfa54f2 is 50, key is test_row_0/C:col10/1733811406257/Put/seqid=0 2024-12-10T06:16:48,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742371_1547 (size=12104) 2024-12-10T06:16:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:48,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a87c189696594b34b34d629f68cd8e2b_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a87c189696594b34b34d629f68cd8e2b_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:48,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/80baeb738e0f4f2594100b0ce86cf2d6, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/80baeb738e0f4f2594100b0ce86cf2d6 is 175, key is test_row_0/A:col10/1733811406305/Put/seqid=0 2024-12-10T06:16:48,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742372_1548 (size=30955) 2024-12-10T06:16:48,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:48,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811468485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:48,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811468487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811468595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811468595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T06:16:48,793 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/f55abdcd1de54d8b9d6e6e8cecfa54f2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/f55abdcd1de54d8b9d6e6e8cecfa54f2 2024-12-10T06:16:48,797 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/C of 112cbd3eaf4337ea756788d925ebbcd9 into f55abdcd1de54d8b9d6e6e8cecfa54f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:48,797 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:48,797 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/C, priority=13, startTime=1733811407927; duration=0sec 2024-12-10T06:16:48,797 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:48,797 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:C 2024-12-10T06:16:48,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811468805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:48,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811468805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:48,849 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/80baeb738e0f4f2594100b0ce86cf2d6 2024-12-10T06:16:48,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/d44183030dc747f58ece836a31530697 is 50, key is test_row_0/B:col10/1733811406305/Put/seqid=0 2024-12-10T06:16:48,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742373_1549 (size=12001) 2024-12-10T06:16:49,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811469115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:49,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811469115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:49,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/d44183030dc747f58ece836a31530697 2024-12-10T06:16:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/54fe0662b3c44ec9b33e388ea147d660 is 50, key is test_row_0/C:col10/1733811406305/Put/seqid=0 2024-12-10T06:16:49,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742374_1550 (size=12001) 2024-12-10T06:16:49,276 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/54fe0662b3c44ec9b33e388ea147d660 2024-12-10T06:16:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/80baeb738e0f4f2594100b0ce86cf2d6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/80baeb738e0f4f2594100b0ce86cf2d6 2024-12-10T06:16:49,282 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/80baeb738e0f4f2594100b0ce86cf2d6, entries=150, sequenceid=83, filesize=30.2 K 2024-12-10T06:16:49,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/d44183030dc747f58ece836a31530697 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d44183030dc747f58ece836a31530697 2024-12-10T06:16:49,286 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d44183030dc747f58ece836a31530697, entries=150, sequenceid=83, filesize=11.7 K 2024-12-10T06:16:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/54fe0662b3c44ec9b33e388ea147d660 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/54fe0662b3c44ec9b33e388ea147d660 2024-12-10T06:16:49,290 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/54fe0662b3c44ec9b33e388ea147d660, entries=150, sequenceid=83, filesize=11.7 K 2024-12-10T06:16:49,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 112cbd3eaf4337ea756788d925ebbcd9 in 1264ms, sequenceid=83, compaction requested=false 2024-12-10T06:16:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-10T06:16:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-10T06:16:49,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-10T06:16:49,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7260 sec 2024-12-10T06:16:49,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.7310 sec 2024-12-10T06:16:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:49,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T06:16:49,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:49,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:49,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:49,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:49,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:49,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:49,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cf37494638c54b8ca76832c59f6d1a88_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811409627/Put/seqid=0 2024-12-10T06:16:49,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742375_1551 (size=14594) 2024-12-10T06:16:49,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811469673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:49,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811469680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T06:16:49,715 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:49,716 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-10T06:16:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-10T06:16:49,718 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:49,718 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:49,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:49,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811469785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:49,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811469795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-10T06:16:49,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:49,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:49,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:49,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811469996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:50,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811470005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:50,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-10T06:16:50,042 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:50,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cf37494638c54b8ca76832c59f6d1a88_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cf37494638c54b8ca76832c59f6d1a88_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:50,046 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/3fe1369b6ff542c78c69922b79cad843, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:50,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/3fe1369b6ff542c78c69922b79cad843 is 175, key is test_row_0/A:col10/1733811409627/Put/seqid=0 2024-12-10T06:16:50,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742376_1552 (size=39549) 2024-12-10T06:16:50,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811470305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:50,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811470315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:50,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-10T06:16:50,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/3fe1369b6ff542c78c69922b79cad843 2024-12-10T06:16:50,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/6743a0170868453d8220e041045c7c0b is 50, key is test_row_0/B:col10/1733811409627/Put/seqid=0 2024-12-10T06:16:50,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742377_1553 (size=12001) 2024-12-10T06:16:50,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:50,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:50,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:50,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:50,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811470816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:50,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811470825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-10T06:16:50,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/6743a0170868453d8220e041045c7c0b 2024-12-10T06:16:50,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/17e3ced6f59f4af497fb64dce9b471fa is 50, key is test_row_0/C:col10/1733811409627/Put/seqid=0 2024-12-10T06:16:50,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742378_1554 (size=12001) 2024-12-10T06:16:50,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:50,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:50,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:50,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:51,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:51,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:51,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:51,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:51,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:51,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:51,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:51,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:51,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/17e3ced6f59f4af497fb64dce9b471fa 2024-12-10T06:16:51,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/3fe1369b6ff542c78c69922b79cad843 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/3fe1369b6ff542c78c69922b79cad843 2024-12-10T06:16:51,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/3fe1369b6ff542c78c69922b79cad843, entries=200, sequenceid=100, filesize=38.6 K 2024-12-10T06:16:51,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/6743a0170868453d8220e041045c7c0b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/6743a0170868453d8220e041045c7c0b 2024-12-10T06:16:51,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/6743a0170868453d8220e041045c7c0b, entries=150, sequenceid=100, filesize=11.7 K 2024-12-10T06:16:51,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/17e3ced6f59f4af497fb64dce9b471fa as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/17e3ced6f59f4af497fb64dce9b471fa 2024-12-10T06:16:51,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/17e3ced6f59f4af497fb64dce9b471fa, entries=150, sequenceid=100, filesize=11.7 K 2024-12-10T06:16:51,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 112cbd3eaf4337ea756788d925ebbcd9 in 1659ms, sequenceid=100, compaction requested=true 2024-12-10T06:16:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:51,288 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:51,288 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:51,289 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:51,289 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/A is initiating minor compaction (all files) 2024-12-10T06:16:51,289 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/A in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:51,289 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/86544fe61f7d45fcb0864fd29140df71, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/80baeb738e0f4f2594100b0ce86cf2d6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/3fe1369b6ff542c78c69922b79cad843] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=99.2 K 2024-12-10T06:16:51,289 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:51,289 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/86544fe61f7d45fcb0864fd29140df71, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/80baeb738e0f4f2594100b0ce86cf2d6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/3fe1369b6ff542c78c69922b79cad843] 2024-12-10T06:16:51,290 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:51,290 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86544fe61f7d45fcb0864fd29140df71, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733811405608 2024-12-10T06:16:51,290 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/B is initiating minor compaction (all files) 2024-12-10T06:16:51,290 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/B in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:51,290 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8c949bfec22b43d9bcd180949142de3d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d44183030dc747f58ece836a31530697, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/6743a0170868453d8220e041045c7c0b] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=35.3 K 2024-12-10T06:16:51,290 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80baeb738e0f4f2594100b0ce86cf2d6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733811406298 2024-12-10T06:16:51,290 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c949bfec22b43d9bcd180949142de3d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733811405608 2024-12-10T06:16:51,290 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fe1369b6ff542c78c69922b79cad843, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733811408471 2024-12-10T06:16:51,290 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d44183030dc747f58ece836a31530697, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733811406298 2024-12-10T06:16:51,291 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 6743a0170868453d8220e041045c7c0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733811408471 2024-12-10T06:16:51,296 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:51,297 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412105304d87877b84c8e8197b5c7ef128286_112cbd3eaf4337ea756788d925ebbcd9 store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:51,297 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:51,298 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/67c39ab7ca8a44829d7f3efe689a0c9a is 50, key is test_row_0/B:col10/1733811409627/Put/seqid=0 2024-12-10T06:16:51,299 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412105304d87877b84c8e8197b5c7ef128286_112cbd3eaf4337ea756788d925ebbcd9, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:51,299 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105304d87877b84c8e8197b5c7ef128286_112cbd3eaf4337ea756788d925ebbcd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:51,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742379_1555 (size=12207) 2024-12-10T06:16:51,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742380_1556 (size=4469) 2024-12-10T06:16:51,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-10T06:16:51,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:51,406 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T06:16:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210709d6fb8c7a2459a87e6ddcfdcf53573_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811409672/Put/seqid=0 2024-12-10T06:16:51,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742381_1557 (size=12154) 2024-12-10T06:16:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:51,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:51,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811471697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:51,706 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#A#compaction#468 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:51,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811471702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:51,707 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/67c39ab7ca8a44829d7f3efe689a0c9a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/67c39ab7ca8a44829d7f3efe689a0c9a 2024-12-10T06:16:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,707 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/b347f4755c524f2bb0e7412573f5ae2a is 175, key is test_row_0/A:col10/1733811409627/Put/seqid=0 2024-12-10T06:16:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:51,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811471703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:51,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742382_1558 (size=31161) 2024-12-10T06:16:51,711 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/B of 112cbd3eaf4337ea756788d925ebbcd9 into 67c39ab7ca8a44829d7f3efe689a0c9a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:51,711 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:51,712 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/B, priority=13, startTime=1733811411288; duration=0sec 2024-12-10T06:16:51,712 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:51,712 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:B 2024-12-10T06:16:51,712 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:51,712 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:51,713 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/C is initiating minor compaction (all files) 2024-12-10T06:16:51,713 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/C in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:51,713 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/f55abdcd1de54d8b9d6e6e8cecfa54f2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/54fe0662b3c44ec9b33e388ea147d660, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/17e3ced6f59f4af497fb64dce9b471fa] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=35.3 K 2024-12-10T06:16:51,713 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting f55abdcd1de54d8b9d6e6e8cecfa54f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733811405608 2024-12-10T06:16:51,713 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 54fe0662b3c44ec9b33e388ea147d660, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733811406298 2024-12-10T06:16:51,714 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 17e3ced6f59f4af497fb64dce9b471fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733811408471 2024-12-10T06:16:51,715 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/b347f4755c524f2bb0e7412573f5ae2a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/b347f4755c524f2bb0e7412573f5ae2a 2024-12-10T06:16:51,719 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/A of 112cbd3eaf4337ea756788d925ebbcd9 into b347f4755c524f2bb0e7412573f5ae2a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:51,719 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:51,719 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/A, priority=13, startTime=1733811411288; duration=0sec 2024-12-10T06:16:51,719 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:51,719 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:A 2024-12-10T06:16:51,719 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:51,720 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/7dbb486028c34dc9a8501ec08bccf08c is 50, key is test_row_0/C:col10/1733811409627/Put/seqid=0 2024-12-10T06:16:51,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742383_1559 (size=12207) 2024-12-10T06:16:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811471805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:51,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:51,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811471815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811471815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:51,820 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210709d6fb8c7a2459a87e6ddcfdcf53573_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210709d6fb8c7a2459a87e6ddcfdcf53573_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:51,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/d8269a6d9cf94e279a77669ed1c785e2, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:51,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/d8269a6d9cf94e279a77669ed1c785e2 is 175, key is test_row_0/A:col10/1733811409672/Put/seqid=0 2024-12-10T06:16:51,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742384_1560 (size=30955) 2024-12-10T06:16:51,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811471825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:51,830 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_1' in 112cbd3eaf4337ea756788d925ebbcd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:51,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811471845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:51,848 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c42c682b34f7,43983,1733811247584 for 'test_row_2' in 112cbd3eaf4337ea756788d925ebbcd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T06:16:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-10T06:16:52,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811472015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:52,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811472025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:52,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:52,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811472026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:52,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:52,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:52,129 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/7dbb486028c34dc9a8501ec08bccf08c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/7dbb486028c34dc9a8501ec08bccf08c 2024-12-10T06:16:52,133 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/C of 112cbd3eaf4337ea756788d925ebbcd9 into 7dbb486028c34dc9a8501ec08bccf08c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:52,133 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:52,133 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/C, priority=13, startTime=1733811411288; duration=0sec 2024-12-10T06:16:52,133 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:52,133 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:C 2024-12-10T06:16:52,225 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/d8269a6d9cf94e279a77669ed1c785e2 2024-12-10T06:16:52,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/50d0a366052a451e80b938a1567c17cc is 50, key is test_row_0/B:col10/1733811409672/Put/seqid=0 2024-12-10T06:16:52,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742385_1561 (size=12001) 2024-12-10T06:16:52,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811472325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:52,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811472335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:52,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811472335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:52,635 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/50d0a366052a451e80b938a1567c17cc 2024-12-10T06:16:52,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/dad987db94374ec18d788b3947c55692 is 50, key is test_row_0/C:col10/1733811409672/Put/seqid=0 2024-12-10T06:16:52,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742386_1562 (size=12001) 2024-12-10T06:16:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811472835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:52,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811472845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811472845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:53,045 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/dad987db94374ec18d788b3947c55692 2024-12-10T06:16:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/d8269a6d9cf94e279a77669ed1c785e2 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/d8269a6d9cf94e279a77669ed1c785e2 2024-12-10T06:16:53,053 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/d8269a6d9cf94e279a77669ed1c785e2, entries=150, sequenceid=118, filesize=30.2 K 2024-12-10T06:16:53,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/50d0a366052a451e80b938a1567c17cc as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/50d0a366052a451e80b938a1567c17cc 2024-12-10T06:16:53,057 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/50d0a366052a451e80b938a1567c17cc, entries=150, sequenceid=118, filesize=11.7 K 2024-12-10T06:16:53,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/dad987db94374ec18d788b3947c55692 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/dad987db94374ec18d788b3947c55692 2024-12-10T06:16:53,061 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/dad987db94374ec18d788b3947c55692, entries=150, sequenceid=118, filesize=11.7 K 2024-12-10T06:16:53,062 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 112cbd3eaf4337ea756788d925ebbcd9 in 1657ms, sequenceid=118, compaction requested=false 2024-12-10T06:16:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-10T06:16:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-10T06:16:53,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-10T06:16:53,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3450 sec 2024-12-10T06:16:53,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 3.3490 sec 2024-12-10T06:16:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:53,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T06:16:53,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:53,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:53,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:53,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:53,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:53,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:53,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210847a99b53e054425ab7bb7a8195a328e_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811411700/Put/seqid=0 2024-12-10T06:16:53,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742387_1563 (size=14794) 2024-12-10T06:16:53,861 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:53,864 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210847a99b53e054425ab7bb7a8195a328e_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210847a99b53e054425ab7bb7a8195a328e_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:53,864 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/cfe06308a2c349e19a24f9b2835c6145, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:53,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/cfe06308a2c349e19a24f9b2835c6145 is 175, key is test_row_0/A:col10/1733811411700/Put/seqid=0 2024-12-10T06:16:53,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742388_1564 (size=39749) 2024-12-10T06:16:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-10T06:16:53,876 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:53,876 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-10T06:16:53,878 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-10T06:16:53,878 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:53,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:53,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811473871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:53,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811473899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811473899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:53,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811473899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:53,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811473901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-10T06:16:54,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811474016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811474016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811474016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811474016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811474016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-10T06:16:54,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:54,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-10T06:16:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-10T06:16:54,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811474225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811474225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811474225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811474225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811474225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,269 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/cfe06308a2c349e19a24f9b2835c6145 2024-12-10T06:16:54,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/d36b89874a88472e821ec03545b79d91 is 50, key is test_row_0/B:col10/1733811411700/Put/seqid=0 2024-12-10T06:16:54,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742389_1565 (size=12151) 2024-12-10T06:16:54,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/d36b89874a88472e821ec03545b79d91 2024-12-10T06:16:54,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/d571ec2d346c45ca87c8ded03bc2cb71 is 50, key is test_row_0/C:col10/1733811411700/Put/seqid=0 2024-12-10T06:16:54,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742390_1566 (size=12151) 2024-12-10T06:16:54,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-10T06:16:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-10T06:16:54,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:54,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-10T06:16:54,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811474535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811474535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811474535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811474535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811474535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:54,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-10T06:16:54,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:54,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:16:54,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/d571ec2d346c45ca87c8ded03bc2cb71 2024-12-10T06:16:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/cfe06308a2c349e19a24f9b2835c6145 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/cfe06308a2c349e19a24f9b2835c6145 2024-12-10T06:16:54,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/cfe06308a2c349e19a24f9b2835c6145, entries=200, sequenceid=137, filesize=38.8 K 2024-12-10T06:16:54,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/d36b89874a88472e821ec03545b79d91 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d36b89874a88472e821ec03545b79d91 2024-12-10T06:16:54,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d36b89874a88472e821ec03545b79d91, entries=150, sequenceid=137, filesize=11.9 K 2024-12-10T06:16:54,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/d571ec2d346c45ca87c8ded03bc2cb71 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/d571ec2d346c45ca87c8ded03bc2cb71 2024-12-10T06:16:54,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/d571ec2d346c45ca87c8ded03bc2cb71, entries=150, sequenceid=137, filesize=11.9 K 2024-12-10T06:16:54,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 112cbd3eaf4337ea756788d925ebbcd9 in 853ms, sequenceid=137, compaction requested=true 2024-12-10T06:16:54,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:54,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:54,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:54,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:54,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:54,703 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:54,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:54,703 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:54,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:54,703 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:54,703 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:54,703 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/B is initiating minor compaction (all files) 2024-12-10T06:16:54,703 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/A is initiating minor compaction (all files) 2024-12-10T06:16:54,704 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/B in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,704 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/A in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,704 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/67c39ab7ca8a44829d7f3efe689a0c9a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/50d0a366052a451e80b938a1567c17cc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d36b89874a88472e821ec03545b79d91] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=35.5 K 2024-12-10T06:16:54,704 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/b347f4755c524f2bb0e7412573f5ae2a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/d8269a6d9cf94e279a77669ed1c785e2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/cfe06308a2c349e19a24f9b2835c6145] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=99.5 K 2024-12-10T06:16:54,704 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,704 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/b347f4755c524f2bb0e7412573f5ae2a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/d8269a6d9cf94e279a77669ed1c785e2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/cfe06308a2c349e19a24f9b2835c6145] 2024-12-10T06:16:54,704 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c39ab7ca8a44829d7f3efe689a0c9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733811408471 2024-12-10T06:16:54,704 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d0a366052a451e80b938a1567c17cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733811409665 2024-12-10T06:16:54,704 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting b347f4755c524f2bb0e7412573f5ae2a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733811408471 2024-12-10T06:16:54,705 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8269a6d9cf94e279a77669ed1c785e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733811409665 2024-12-10T06:16:54,705 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d36b89874a88472e821ec03545b79d91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733811411695 2024-12-10T06:16:54,705 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe06308a2c349e19a24f9b2835c6145, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733811411695 2024-12-10T06:16:54,715 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:54,716 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/bc842251cacb493aaedf8e9d39cb72cf is 50, key is test_row_0/B:col10/1733811411700/Put/seqid=0 2024-12-10T06:16:54,718 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:54,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742391_1567 (size=12459) 2024-12-10T06:16:54,721 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412103337249456334e0f9fade0add29c8120_112cbd3eaf4337ea756788d925ebbcd9 store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:54,722 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412103337249456334e0f9fade0add29c8120_112cbd3eaf4337ea756788d925ebbcd9, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:54,723 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103337249456334e0f9fade0add29c8120_112cbd3eaf4337ea756788d925ebbcd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:54,725 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/bc842251cacb493aaedf8e9d39cb72cf as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/bc842251cacb493aaedf8e9d39cb72cf 2024-12-10T06:16:54,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742392_1568 (size=4469) 2024-12-10T06:16:54,729 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/B of 112cbd3eaf4337ea756788d925ebbcd9 into bc842251cacb493aaedf8e9d39cb72cf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:54,729 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:54,729 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/B, priority=13, startTime=1733811414703; duration=0sec 2024-12-10T06:16:54,729 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:54,729 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:B 2024-12-10T06:16:54,729 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:54,730 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:54,730 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/C is initiating minor compaction (all files) 2024-12-10T06:16:54,730 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/C in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,730 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/7dbb486028c34dc9a8501ec08bccf08c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/dad987db94374ec18d788b3947c55692, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/d571ec2d346c45ca87c8ded03bc2cb71] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=35.5 K 2024-12-10T06:16:54,730 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dbb486028c34dc9a8501ec08bccf08c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733811408471 2024-12-10T06:16:54,731 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting dad987db94374ec18d788b3947c55692, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733811409665 2024-12-10T06:16:54,731 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting d571ec2d346c45ca87c8ded03bc2cb71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733811411695 2024-12-10T06:16:54,736 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:54,737 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/b449417ba1054c85bca6c2aac37607ef is 50, key is test_row_0/C:col10/1733811411700/Put/seqid=0 2024-12-10T06:16:54,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742393_1569 (size=12459) 2024-12-10T06:16:54,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-10T06:16:54,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:54,795 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T06:16:54,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:54,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:54,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:54,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:54,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:54,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:54,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210748feacdcad24fdc95257e05bab73708_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811413899/Put/seqid=0 2024-12-10T06:16:54,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742394_1570 (size=12304) 2024-12-10T06:16:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-10T06:16:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:55,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811475058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811475059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811475062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811475064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811475065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,127 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:55,128 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/fe69a34cc86241a0a2006a176049b129 is 175, key is test_row_0/A:col10/1733811411700/Put/seqid=0 2024-12-10T06:16:55,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742395_1571 (size=31413) 2024-12-10T06:16:55,144 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/b449417ba1054c85bca6c2aac37607ef as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b449417ba1054c85bca6c2aac37607ef 2024-12-10T06:16:55,148 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/C of 112cbd3eaf4337ea756788d925ebbcd9 into b449417ba1054c85bca6c2aac37607ef(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:55,148 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:55,148 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/C, priority=13, startTime=1733811414703; duration=0sec 2024-12-10T06:16:55,148 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:55,148 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:C 2024-12-10T06:16:55,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811475176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811475176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811475176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811475176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811475176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:55,208 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210748feacdcad24fdc95257e05bab73708_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210748feacdcad24fdc95257e05bab73708_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/72f472e072b045ad90e9650a2bb19512, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/72f472e072b045ad90e9650a2bb19512 is 175, key is test_row_0/A:col10/1733811413899/Put/seqid=0 2024-12-10T06:16:55,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742396_1572 (size=31105) 2024-12-10T06:16:55,214 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/72f472e072b045ad90e9650a2bb19512 2024-12-10T06:16:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8ea6c65c6cbe4c9cba02565530002833 is 50, key is test_row_0/B:col10/1733811413899/Put/seqid=0 2024-12-10T06:16:55,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742397_1573 (size=12151) 2024-12-10T06:16:55,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811475385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811475385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811475385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811475385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811475385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,536 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/fe69a34cc86241a0a2006a176049b129 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/fe69a34cc86241a0a2006a176049b129 2024-12-10T06:16:55,540 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/A of 112cbd3eaf4337ea756788d925ebbcd9 into fe69a34cc86241a0a2006a176049b129(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:55,540 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:55,540 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/A, priority=13, startTime=1733811414702; duration=0sec 2024-12-10T06:16:55,541 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:55,541 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:A 2024-12-10T06:16:55,624 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8ea6c65c6cbe4c9cba02565530002833 2024-12-10T06:16:55,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/bc335f2bd8134bf681913879f6d0414d is 50, key is test_row_0/C:col10/1733811413899/Put/seqid=0 2024-12-10T06:16:55,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742398_1574 (size=12151) 2024-12-10T06:16:55,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811475696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811475696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811475696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811475696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811475696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-10T06:16:56,035 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/bc335f2bd8134bf681913879f6d0414d 2024-12-10T06:16:56,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/72f472e072b045ad90e9650a2bb19512 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/72f472e072b045ad90e9650a2bb19512 2024-12-10T06:16:56,043 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/72f472e072b045ad90e9650a2bb19512, entries=150, sequenceid=154, filesize=30.4 K 2024-12-10T06:16:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8ea6c65c6cbe4c9cba02565530002833 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8ea6c65c6cbe4c9cba02565530002833 2024-12-10T06:16:56,047 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8ea6c65c6cbe4c9cba02565530002833, entries=150, sequenceid=154, filesize=11.9 K 2024-12-10T06:16:56,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/bc335f2bd8134bf681913879f6d0414d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/bc335f2bd8134bf681913879f6d0414d 2024-12-10T06:16:56,050 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/bc335f2bd8134bf681913879f6d0414d, entries=150, sequenceid=154, filesize=11.9 K 2024-12-10T06:16:56,051 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 112cbd3eaf4337ea756788d925ebbcd9 in 1256ms, sequenceid=154, compaction requested=false 2024-12-10T06:16:56,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:56,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:56,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-10T06:16:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-10T06:16:56,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-10T06:16:56,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-12-10T06:16:56,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.1790 sec 2024-12-10T06:16:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:56,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T06:16:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:56,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105f6c331c75f943f0999a8c1e372cba32_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811415064/Put/seqid=0 2024-12-10T06:16:56,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742399_1575 (size=17284) 2024-12-10T06:16:56,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811476222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811476224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811476228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811476230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811476231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811476335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811476335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811476335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811476345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811476345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811476545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811476545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811476545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811476556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811476556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,623 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:56,630 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105f6c331c75f943f0999a8c1e372cba32_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105f6c331c75f943f0999a8c1e372cba32_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:56,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/ed47d912b57c412fb349375cf725b98b, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:56,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/ed47d912b57c412fb349375cf725b98b is 175, key is test_row_0/A:col10/1733811415064/Put/seqid=0 2024-12-10T06:16:56,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742400_1576 (size=48389) 2024-12-10T06:16:56,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811476855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811476855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811476855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811476866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811476866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:57,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/ed47d912b57c412fb349375cf725b98b 2024-12-10T06:16:57,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/33e18a68743d4d33b3e3df9169f24734 is 50, key is test_row_0/B:col10/1733811415064/Put/seqid=0 2024-12-10T06:16:57,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742401_1577 (size=12151) 2024-12-10T06:16:57,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811477366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:57,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811477366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:57,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811477366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:57,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811477375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:57,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811477385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:57,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/33e18a68743d4d33b3e3df9169f24734 2024-12-10T06:16:57,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/2909c4c9b250435199a2f448925f6cb8 is 50, key is test_row_0/C:col10/1733811415064/Put/seqid=0 2024-12-10T06:16:57,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742402_1578 (size=12151) 2024-12-10T06:16:57,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/2909c4c9b250435199a2f448925f6cb8 2024-12-10T06:16:57,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/ed47d912b57c412fb349375cf725b98b as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/ed47d912b57c412fb349375cf725b98b 2024-12-10T06:16:57,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/ed47d912b57c412fb349375cf725b98b, entries=250, sequenceid=175, filesize=47.3 K 2024-12-10T06:16:57,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/33e18a68743d4d33b3e3df9169f24734 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/33e18a68743d4d33b3e3df9169f24734 2024-12-10T06:16:57,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/33e18a68743d4d33b3e3df9169f24734, entries=150, sequenceid=175, filesize=11.9 K 2024-12-10T06:16:57,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/2909c4c9b250435199a2f448925f6cb8 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/2909c4c9b250435199a2f448925f6cb8 2024-12-10T06:16:57,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/2909c4c9b250435199a2f448925f6cb8, entries=150, sequenceid=175, filesize=11.9 K 2024-12-10T06:16:57,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 112cbd3eaf4337ea756788d925ebbcd9 in 1357ms, sequenceid=175, compaction requested=true 2024-12-10T06:16:57,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:57,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:16:57,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:16:57,566 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:16:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T06:16:57,566 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:57,566 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:57,567 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:57,567 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/B is initiating minor compaction (all files) 2024-12-10T06:16:57,567 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/A is initiating minor compaction (all files) 2024-12-10T06:16:57,567 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/B in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:57,567 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/A in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:57,567 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/bc842251cacb493aaedf8e9d39cb72cf, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8ea6c65c6cbe4c9cba02565530002833, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/33e18a68743d4d33b3e3df9169f24734] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=35.9 K 2024-12-10T06:16:57,567 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/fe69a34cc86241a0a2006a176049b129, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/72f472e072b045ad90e9650a2bb19512, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/ed47d912b57c412fb349375cf725b98b] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=108.3 K 2024-12-10T06:16:57,567 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:57,567 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/fe69a34cc86241a0a2006a176049b129, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/72f472e072b045ad90e9650a2bb19512, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/ed47d912b57c412fb349375cf725b98b] 2024-12-10T06:16:57,567 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting bc842251cacb493aaedf8e9d39cb72cf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733811411695 2024-12-10T06:16:57,567 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe69a34cc86241a0a2006a176049b129, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733811411695 2024-12-10T06:16:57,568 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea6c65c6cbe4c9cba02565530002833, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733811413863 2024-12-10T06:16:57,568 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72f472e072b045ad90e9650a2bb19512, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733811413863 2024-12-10T06:16:57,569 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed47d912b57c412fb349375cf725b98b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733811415062 2024-12-10T06:16:57,569 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 33e18a68743d4d33b3e3df9169f24734, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733811415062 2024-12-10T06:16:57,581 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:57,585 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412103086673026ca4e3fb7fcfa0b08463791_112cbd3eaf4337ea756788d925ebbcd9 store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:57,586 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#B#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:57,586 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/7bd79898900c4f0e9e9863af0d6d280c is 50, key is test_row_0/B:col10/1733811415064/Put/seqid=0 2024-12-10T06:16:57,587 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412103086673026ca4e3fb7fcfa0b08463791_112cbd3eaf4337ea756788d925ebbcd9, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:57,588 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103086673026ca4e3fb7fcfa0b08463791_112cbd3eaf4337ea756788d925ebbcd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:57,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742403_1579 (size=12561) 2024-12-10T06:16:57,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742404_1580 (size=4469) 2024-12-10T06:16:58,000 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/7bd79898900c4f0e9e9863af0d6d280c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/7bd79898900c4f0e9e9863af0d6d280c 2024-12-10T06:16:58,004 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/B of 112cbd3eaf4337ea756788d925ebbcd9 into 7bd79898900c4f0e9e9863af0d6d280c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:58,004 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:58,004 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/B, priority=13, startTime=1733811417565; duration=0sec 2024-12-10T06:16:58,004 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:16:58,004 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:B 2024-12-10T06:16:58,004 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T06:16:58,005 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T06:16:58,005 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/C is initiating minor compaction (all files) 2024-12-10T06:16:58,005 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/C in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:58,005 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b449417ba1054c85bca6c2aac37607ef, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/bc335f2bd8134bf681913879f6d0414d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/2909c4c9b250435199a2f448925f6cb8] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=35.9 K 2024-12-10T06:16:58,005 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b449417ba1054c85bca6c2aac37607ef, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733811411695 2024-12-10T06:16:58,006 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting bc335f2bd8134bf681913879f6d0414d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733811413863 2024-12-10T06:16:58,006 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 2909c4c9b250435199a2f448925f6cb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733811415062 2024-12-10T06:16:58,011 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#A#compaction#486 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:58,012 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/6bb6609b08e3476d8e64f93f40e30ae7 is 175, key is test_row_0/A:col10/1733811415064/Put/seqid=0 2024-12-10T06:16:58,014 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:16:58,014 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/589ffbacc8a14a38ad00b4dd7aa58481 is 50, key is test_row_0/C:col10/1733811415064/Put/seqid=0 2024-12-10T06:16:58,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742405_1581 (size=31515) 2024-12-10T06:16:58,022 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/6bb6609b08e3476d8e64f93f40e30ae7 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6bb6609b08e3476d8e64f93f40e30ae7 2024-12-10T06:16:58,027 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/A of 112cbd3eaf4337ea756788d925ebbcd9 into 6bb6609b08e3476d8e64f93f40e30ae7(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:58,027 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:58,027 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/A, priority=13, startTime=1733811417565; duration=0sec 2024-12-10T06:16:58,027 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:58,027 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:A 2024-12-10T06:16:58,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742406_1582 (size=12561) 2024-12-10T06:16:58,031 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/589ffbacc8a14a38ad00b4dd7aa58481 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/589ffbacc8a14a38ad00b4dd7aa58481 2024-12-10T06:16:58,036 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/C of 112cbd3eaf4337ea756788d925ebbcd9 into 589ffbacc8a14a38ad00b4dd7aa58481(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:16:58,037 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:58,037 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/C, priority=13, startTime=1733811417566; duration=0sec 2024-12-10T06:16:58,037 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:16:58,037 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:C 2024-12-10T06:16:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-10T06:16:58,046 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:58,046 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-10T06:16:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-10T06:16:58,049 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:58,049 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:58,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-10T06:16:58,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-10T06:16:58,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:58,202 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T06:16:58,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:58,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:58,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:58,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:58,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:58,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b52c8ac8a5ca4467839972d8f1abc3a4_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811416226/Put/seqid=0 2024-12-10T06:16:58,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742407_1583 (size=12304) 2024-12-10T06:16:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-10T06:16:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:58,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811478409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811478410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811478413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811478415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811478415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811478525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811478525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811478525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811478525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811478525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:58,645 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b52c8ac8a5ca4467839972d8f1abc3a4_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b52c8ac8a5ca4467839972d8f1abc3a4_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:58,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/4e616a377abe47d086b93c41f904bece, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/4e616a377abe47d086b93c41f904bece is 175, key is test_row_0/A:col10/1733811416226/Put/seqid=0 2024-12-10T06:16:58,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742408_1584 (size=31105) 2024-12-10T06:16:58,656 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/4e616a377abe47d086b93c41f904bece 2024-12-10T06:16:58,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/05939baeaf8848c8bb1ad7c221f93b67 is 50, key is test_row_0/B:col10/1733811416226/Put/seqid=0 2024-12-10T06:16:58,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742409_1585 (size=12151) 2024-12-10T06:16:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-10T06:16:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811478735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811478736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811478736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811478736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811478736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811479046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811479046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811479046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811479046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811479046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,075 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/05939baeaf8848c8bb1ad7c221f93b67 2024-12-10T06:16:59,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/92383256df3744b9bea7493743fb6b35 is 50, key is test_row_0/C:col10/1733811416226/Put/seqid=0 2024-12-10T06:16:59,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742410_1586 (size=12151) 2024-12-10T06:16:59,091 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/92383256df3744b9bea7493743fb6b35 2024-12-10T06:16:59,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/4e616a377abe47d086b93c41f904bece as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/4e616a377abe47d086b93c41f904bece 2024-12-10T06:16:59,104 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/4e616a377abe47d086b93c41f904bece, entries=150, sequenceid=192, filesize=30.4 K 2024-12-10T06:16:59,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/05939baeaf8848c8bb1ad7c221f93b67 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/05939baeaf8848c8bb1ad7c221f93b67 2024-12-10T06:16:59,125 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/05939baeaf8848c8bb1ad7c221f93b67, entries=150, sequenceid=192, filesize=11.9 K 2024-12-10T06:16:59,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/92383256df3744b9bea7493743fb6b35 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/92383256df3744b9bea7493743fb6b35 2024-12-10T06:16:59,133 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/92383256df3744b9bea7493743fb6b35, entries=150, sequenceid=192, filesize=11.9 K 2024-12-10T06:16:59,134 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 112cbd3eaf4337ea756788d925ebbcd9 in 932ms, sequenceid=192, compaction requested=false 2024-12-10T06:16:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:16:59,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:59,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-10T06:16:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-10T06:16:59,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-10T06:16:59,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2024-12-10T06:16:59,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0930 sec 2024-12-10T06:16:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-10T06:16:59,186 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:16:59,187 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:16:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-10T06:16:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T06:16:59,190 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:16:59,190 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:16:59,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:16:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T06:16:59,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-10T06:16:59,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:16:59,345 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T06:16:59,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:16:59,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:59,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:16:59,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:59,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:16:59,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:16:59,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210aa6d84c5277349ff8665ad9e1992fa89_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811418411/Put/seqid=0 2024-12-10T06:16:59,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742411_1587 (size=12304) 2024-12-10T06:16:59,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:16:59,372 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210aa6d84c5277349ff8665ad9e1992fa89_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210aa6d84c5277349ff8665ad9e1992fa89_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:59,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/23ea383111a64f71b3ca06771575551a, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:16:59,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/23ea383111a64f71b3ca06771575551a is 175, key is test_row_0/A:col10/1733811418411/Put/seqid=0 2024-12-10T06:16:59,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742412_1588 (size=31105) 2024-12-10T06:16:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T06:16:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:16:59,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:16:59,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811479568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811479569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811479570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811479571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811479577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811479685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811479685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811479685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811479685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811479685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,784 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/23ea383111a64f71b3ca06771575551a 2024-12-10T06:16:59,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/ff2206daa0ba480d80d3298d730d8372 is 50, key is test_row_0/B:col10/1733811418411/Put/seqid=0 2024-12-10T06:16:59,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742413_1589 (size=12151) 2024-12-10T06:16:59,814 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/ff2206daa0ba480d80d3298d730d8372 2024-12-10T06:16:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T06:16:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/6f0e900d36f6402da10a7c11abcc01c6 is 50, key is test_row_0/C:col10/1733811418411/Put/seqid=0 2024-12-10T06:16:59,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742414_1590 (size=12151) 2024-12-10T06:16:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811479896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811479896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811479896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811479896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:16:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811479896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:16:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:16:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:16:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811480205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811480205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811480205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811480206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811480206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,230 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/6f0e900d36f6402da10a7c11abcc01c6 2024-12-10T06:17:00,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/23ea383111a64f71b3ca06771575551a as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/23ea383111a64f71b3ca06771575551a 2024-12-10T06:17:00,237 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/23ea383111a64f71b3ca06771575551a, entries=150, sequenceid=211, filesize=30.4 K 2024-12-10T06:17:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/ff2206daa0ba480d80d3298d730d8372 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ff2206daa0ba480d80d3298d730d8372 2024-12-10T06:17:00,242 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ff2206daa0ba480d80d3298d730d8372, entries=150, sequenceid=211, filesize=11.9 K 2024-12-10T06:17:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/6f0e900d36f6402da10a7c11abcc01c6 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/6f0e900d36f6402da10a7c11abcc01c6 2024-12-10T06:17:00,246 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/6f0e900d36f6402da10a7c11abcc01c6, entries=150, sequenceid=211, filesize=11.9 K 2024-12-10T06:17:00,247 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 112cbd3eaf4337ea756788d925ebbcd9 in 902ms, sequenceid=211, compaction requested=true 2024-12-10T06:17:00,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:17:00,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:00,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-10T06:17:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-10T06:17:00,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-10T06:17:00,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0580 sec 2024-12-10T06:17:00,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0640 sec 2024-12-10T06:17:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T06:17:00,325 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:17:00,326 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:17:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-10T06:17:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T06:17:00,327 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:17:00,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:17:00,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:17:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T06:17:00,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-10T06:17:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:00,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T06:17:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:17:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:17:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:17:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:17:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:17:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:17:00,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210932e707909f24d58bc5a5ac517fab0e2_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811419569/Put/seqid=0 2024-12-10T06:17:00,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742415_1591 (size=12304) 2024-12-10T06:17:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T06:17:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:00,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:17:00,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811480736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811480736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811480737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811480743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811480742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811480856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811480856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811480856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811480856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811480856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:00,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:17:00,896 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210932e707909f24d58bc5a5ac517fab0e2_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210932e707909f24d58bc5a5ac517fab0e2_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:00,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/16298138185645e8a7ef30c4f1b842d1, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:17:00,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/16298138185645e8a7ef30c4f1b842d1 is 175, key is test_row_0/A:col10/1733811419569/Put/seqid=0 2024-12-10T06:17:00,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742416_1592 (size=31105) 2024-12-10T06:17:00,907 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/16298138185645e8a7ef30c4f1b842d1 2024-12-10T06:17:00,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8661487b985d41be99dba236253abce4 is 50, key is test_row_0/B:col10/1733811419569/Put/seqid=0 2024-12-10T06:17:00,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742417_1593 (size=12151) 2024-12-10T06:17:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T06:17:01,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811481065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811481065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811481065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811481065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811481065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,317 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8661487b985d41be99dba236253abce4 2024-12-10T06:17:01,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/b638556318fe4e65b528aecfae802105 is 50, key is test_row_0/C:col10/1733811419569/Put/seqid=0 2024-12-10T06:17:01,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742418_1594 (size=12151) 2024-12-10T06:17:01,335 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/b638556318fe4e65b528aecfae802105 2024-12-10T06:17:01,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/16298138185645e8a7ef30c4f1b842d1 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/16298138185645e8a7ef30c4f1b842d1 2024-12-10T06:17:01,345 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/16298138185645e8a7ef30c4f1b842d1, entries=150, sequenceid=224, filesize=30.4 K 2024-12-10T06:17:01,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8661487b985d41be99dba236253abce4 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8661487b985d41be99dba236253abce4 2024-12-10T06:17:01,351 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8661487b985d41be99dba236253abce4, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T06:17:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/b638556318fe4e65b528aecfae802105 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b638556318fe4e65b528aecfae802105 2024-12-10T06:17:01,355 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b638556318fe4e65b528aecfae802105, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T06:17:01,356 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 112cbd3eaf4337ea756788d925ebbcd9 in 875ms, sequenceid=224, compaction requested=true 2024-12-10T06:17:01,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:17:01,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:01,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-10T06:17:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-10T06:17:01,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-10T06:17:01,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0290 sec 2024-12-10T06:17:01,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0340 sec 2024-12-10T06:17:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:01,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-12-10T06:17:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:17:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:17:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:17:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:17:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:17:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T06:17:01,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210aba4eb5c2fe1416d8fdd3968ad518b18_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811421380/Put/seqid=0 2024-12-10T06:17:01,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811481414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811481415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811481416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811481418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811481418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742419_1595 (size=12304) 2024-12-10T06:17:01,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:17:01,430 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210aba4eb5c2fe1416d8fdd3968ad518b18_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210aba4eb5c2fe1416d8fdd3968ad518b18_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:01,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/6f27598a44d94085b6c81d42f5a43018, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:17:01,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/6f27598a44d94085b6c81d42f5a43018 is 175, key is test_row_0/A:col10/1733811421380/Put/seqid=0 2024-12-10T06:17:01,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742420_1596 (size=31105) 2024-12-10T06:17:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T06:17:01,465 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:17:01,466 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T06:17:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-10T06:17:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T06:17:01,468 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T06:17:01,468 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T06:17:01,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T06:17:01,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811481526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811481526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811481535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811481535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811481535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T06:17:01,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T06:17:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:17:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:01,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:01,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811481735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811481735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811481745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811481745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811481745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:01,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T06:17:01,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:01,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:17:01,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:01,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:01,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T06:17:01,839 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/6f27598a44d94085b6c81d42f5a43018 2024-12-10T06:17:01,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/028c501ac215417793bc9dfcbbc58730 is 50, key is test_row_0/B:col10/1733811421380/Put/seqid=0 2024-12-10T06:17:01,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742421_1597 (size=12151) 2024-12-10T06:17:01,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T06:17:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:17:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:01,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811482045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811482045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811482055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811482055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811482055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T06:17:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:17:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T06:17:02,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T06:17:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:17:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/028c501ac215417793bc9dfcbbc58730 2024-12-10T06:17:02,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/99e32b8bb4b84f9f843df805d8b2b23c is 50, key is test_row_0/C:col10/1733811421380/Put/seqid=0 2024-12-10T06:17:02,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742422_1598 (size=12151) 2024-12-10T06:17:02,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T06:17:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:17:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T06:17:02,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:17:02,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:17:02,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1733811482555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1733811482566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35096 deadline: 1733811482566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1733811482566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1733811482566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 , the old value is region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T06:17:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., hostname=c42c682b34f7,43983,1733811247584, seqNum=5 because the exception is null or not the one we care about 2024-12-10T06:17:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T06:17:02,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/99e32b8bb4b84f9f843df805d8b2b23c 2024-12-10T06:17:02,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/6f27598a44d94085b6c81d42f5a43018 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6f27598a44d94085b6c81d42f5a43018 2024-12-10T06:17:02,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6f27598a44d94085b6c81d42f5a43018, entries=150, sequenceid=247, filesize=30.4 K 2024-12-10T06:17:02,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/028c501ac215417793bc9dfcbbc58730 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/028c501ac215417793bc9dfcbbc58730 2024-12-10T06:17:02,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/028c501ac215417793bc9dfcbbc58730, entries=150, sequenceid=247, filesize=11.9 K 2024-12-10T06:17:02,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/99e32b8bb4b84f9f843df805d8b2b23c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/99e32b8bb4b84f9f843df805d8b2b23c 2024-12-10T06:17:02,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/99e32b8bb4b84f9f843df805d8b2b23c, entries=150, sequenceid=247, filesize=11.9 K 2024-12-10T06:17:02,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=46.96 KB/48090 for 112cbd3eaf4337ea756788d925ebbcd9 in 1271ms, sequenceid=247, compaction requested=true 2024-12-10T06:17:02,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:17:02,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T06:17:02,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:17:02,678 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T06:17:02,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T06:17:02,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:17:02,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 112cbd3eaf4337ea756788d925ebbcd9:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T06:17:02,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:17:02,678 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T06:17:02,679 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 155935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T06:17:02,679 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T06:17:02,679 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/A is initiating minor compaction (all files) 2024-12-10T06:17:02,679 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/A in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,679 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/B is initiating minor compaction (all files) 2024-12-10T06:17:02,679 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/B in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,679 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6bb6609b08e3476d8e64f93f40e30ae7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/4e616a377abe47d086b93c41f904bece, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/23ea383111a64f71b3ca06771575551a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/16298138185645e8a7ef30c4f1b842d1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6f27598a44d94085b6c81d42f5a43018] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=152.3 K 2024-12-10T06:17:02,679 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,679 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/7bd79898900c4f0e9e9863af0d6d280c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/05939baeaf8848c8bb1ad7c221f93b67, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ff2206daa0ba480d80d3298d730d8372, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8661487b985d41be99dba236253abce4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/028c501ac215417793bc9dfcbbc58730] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=59.7 K 2024-12-10T06:17:02,679 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. files: [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6bb6609b08e3476d8e64f93f40e30ae7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/4e616a377abe47d086b93c41f904bece, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/23ea383111a64f71b3ca06771575551a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/16298138185645e8a7ef30c4f1b842d1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6f27598a44d94085b6c81d42f5a43018] 2024-12-10T06:17:02,680 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bb6609b08e3476d8e64f93f40e30ae7, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733811415062 2024-12-10T06:17:02,680 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd79898900c4f0e9e9863af0d6d280c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733811415062 2024-12-10T06:17:02,680 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e616a377abe47d086b93c41f904bece, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733811416226 2024-12-10T06:17:02,680 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 05939baeaf8848c8bb1ad7c221f93b67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733811416226 2024-12-10T06:17:02,680 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ea383111a64f71b3ca06771575551a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733811418407 2024-12-10T06:17:02,680 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting ff2206daa0ba480d80d3298d730d8372, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733811418407 2024-12-10T06:17:02,681 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16298138185645e8a7ef30c4f1b842d1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733811419566 2024-12-10T06:17:02,681 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 8661487b985d41be99dba236253abce4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733811419566 2024-12-10T06:17:02,681 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f27598a44d94085b6c81d42f5a43018, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733811421378 2024-12-10T06:17:02,681 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 028c501ac215417793bc9dfcbbc58730, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733811421378 2024-12-10T06:17:02,689 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:17:02,691 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#B#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:17:02,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T06:17:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,691 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/18a34146ce254501a3a2fd5e5394b02d is 50, key is test_row_0/B:col10/1733811421380/Put/seqid=0 2024-12-10T06:17:02,691 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T06:17:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:17:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:17:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:17:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:17:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:17:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:17:02,693 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412106a43aaf1d21c47e29071d9ec70fc551a_112cbd3eaf4337ea756788d925ebbcd9 store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:17:02,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742423_1599 (size=12731) 2024-12-10T06:17:02,697 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412106a43aaf1d21c47e29071d9ec70fc551a_112cbd3eaf4337ea756788d925ebbcd9, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:17:02,697 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106a43aaf1d21c47e29071d9ec70fc551a_112cbd3eaf4337ea756788d925ebbcd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:17:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b74dbf51044048d3aa698a144c2d7f06_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811421412/Put/seqid=0 2024-12-10T06:17:02,702 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/18a34146ce254501a3a2fd5e5394b02d as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/18a34146ce254501a3a2fd5e5394b02d 2024-12-10T06:17:02,707 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/B of 112cbd3eaf4337ea756788d925ebbcd9 into 18a34146ce254501a3a2fd5e5394b02d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:17:02,707 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:17:02,707 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/B, priority=11, startTime=1733811422678; duration=0sec 2024-12-10T06:17:02,707 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T06:17:02,707 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:B 2024-12-10T06:17:02,707 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T06:17:02,708 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T06:17:02,708 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1541): 112cbd3eaf4337ea756788d925ebbcd9/C is initiating minor compaction (all files) 2024-12-10T06:17:02,708 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 112cbd3eaf4337ea756788d925ebbcd9/C in TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:02,708 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/589ffbacc8a14a38ad00b4dd7aa58481, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/92383256df3744b9bea7493743fb6b35, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/6f0e900d36f6402da10a7c11abcc01c6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b638556318fe4e65b528aecfae802105, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/99e32b8bb4b84f9f843df805d8b2b23c] into tmpdir=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp, totalSize=59.7 K 2024-12-10T06:17:02,709 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 589ffbacc8a14a38ad00b4dd7aa58481, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733811415062 2024-12-10T06:17:02,709 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 92383256df3744b9bea7493743fb6b35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733811416226 2024-12-10T06:17:02,709 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f0e900d36f6402da10a7c11abcc01c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733811418407 2024-12-10T06:17:02,709 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting b638556318fe4e65b528aecfae802105, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733811419566 2024-12-10T06:17:02,710 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e32b8bb4b84f9f843df805d8b2b23c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733811421378 2024-12-10T06:17:02,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742424_1600 (size=4469) 2024-12-10T06:17:02,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742425_1601 (size=12304) 2024-12-10T06:17:02,726 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#C#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:17:02,727 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/850b043860c448069e28a4cbaf82a574 is 50, key is test_row_0/C:col10/1733811421380/Put/seqid=0 2024-12-10T06:17:02,728 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 112cbd3eaf4337ea756788d925ebbcd9#A#compaction#502 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:17:02,729 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/32a2fdedfa674eaca1ad7050620bc17c is 175, key is test_row_0/A:col10/1733811421380/Put/seqid=0 2024-12-10T06:17:02,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:17:02,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742427_1603 (size=31685) 2024-12-10T06:17:02,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742426_1602 (size=12731) 2024-12-10T06:17:02,738 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b74dbf51044048d3aa698a144c2d7f06_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b74dbf51044048d3aa698a144c2d7f06_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:02,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/31f61521b6d74f52b44db23a91eabbe7, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:17:02,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/31f61521b6d74f52b44db23a91eabbe7 is 175, key is test_row_0/A:col10/1733811421412/Put/seqid=0 2024-12-10T06:17:02,742 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/32a2fdedfa674eaca1ad7050620bc17c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/32a2fdedfa674eaca1ad7050620bc17c 2024-12-10T06:17:02,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742428_1604 (size=31105) 2024-12-10T06:17:02,749 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/A of 112cbd3eaf4337ea756788d925ebbcd9 into 32a2fdedfa674eaca1ad7050620bc17c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:17:02,749 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:17:02,749 INFO [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/A, priority=11, startTime=1733811422678; duration=0sec 2024-12-10T06:17:02,749 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:17:02,749 DEBUG [RS:0;c42c682b34f7:43983-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:A 2024-12-10T06:17:03,143 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/850b043860c448069e28a4cbaf82a574 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/850b043860c448069e28a4cbaf82a574 2024-12-10T06:17:03,146 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/31f61521b6d74f52b44db23a91eabbe7 2024-12-10T06:17:03,147 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 112cbd3eaf4337ea756788d925ebbcd9/C of 112cbd3eaf4337ea756788d925ebbcd9 into 850b043860c448069e28a4cbaf82a574(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:17:03,148 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:17:03,148 INFO [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., storeName=112cbd3eaf4337ea756788d925ebbcd9/C, priority=11, startTime=1733811422678; duration=0sec 2024-12-10T06:17:03,148 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T06:17:03,148 DEBUG [RS:0;c42c682b34f7:43983-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 112cbd3eaf4337ea756788d925ebbcd9:C 2024-12-10T06:17:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/d82c0ad313d84388957dfe0aa246a3d3 is 50, key is test_row_0/B:col10/1733811421412/Put/seqid=0 2024-12-10T06:17:03,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742429_1605 (size=12151) 2024-12-10T06:17:03,399 INFO [Thread-2386 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2386. 2024-12-10T06:17:03,399 DEBUG [Thread-2386 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,399 DEBUG [Thread-2386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,399 DEBUG [Thread-2386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,399 INFO [Thread-2384 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2384. 2024-12-10T06:17:03,400 DEBUG [Thread-2384 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,400 DEBUG [Thread-2384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,400 DEBUG [Thread-2384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,400 INFO [Thread-2382 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2382. 2024-12-10T06:17:03,400 DEBUG [Thread-2382 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,400 INFO [Thread-2388 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2388. 2024-12-10T06:17:03,400 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,400 DEBUG [Thread-2388 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,400 DEBUG [Thread-2388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,400 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,400 INFO [Thread-2380 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2380. 2024-12-10T06:17:03,400 DEBUG [Thread-2388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,400 DEBUG [Thread-2380 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,400 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,400 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,556 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/d82c0ad313d84388957dfe0aa246a3d3 2024-12-10T06:17:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/aac6589e105f4857abc2822a1ab22001 is 50, key is test_row_0/C:col10/1733811421412/Put/seqid=0 2024-12-10T06:17:03,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742430_1606 (size=12151) 2024-12-10T06:17:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43983 {}] regionserver.HRegion(8855): Flush requested on 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:03,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. as already flushing 2024-12-10T06:17:03,577 INFO [Thread-2371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2371. 2024-12-10T06:17:03,577 INFO [Thread-2373 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2373. 2024-12-10T06:17:03,577 DEBUG [Thread-2371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,577 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,577 DEBUG [Thread-2373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,577 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,577 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,577 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,577 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,577 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,587 INFO [Thread-2369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2369. 2024-12-10T06:17:03,587 INFO [Thread-2375 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2375. 2024-12-10T06:17:03,587 DEBUG [Thread-2369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,587 INFO [Thread-2377 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2377. 2024-12-10T06:17:03,587 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,587 DEBUG [Thread-2375 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,587 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,587 DEBUG [Thread-2377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T06:17:03,587 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,587 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,587 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,587 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:03,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T06:17:03,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/aac6589e105f4857abc2822a1ab22001 2024-12-10T06:17:03,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/31f61521b6d74f52b44db23a91eabbe7 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/31f61521b6d74f52b44db23a91eabbe7 2024-12-10T06:17:03,971 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/31f61521b6d74f52b44db23a91eabbe7, entries=150, sequenceid=257, filesize=30.4 K 2024-12-10T06:17:03,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/d82c0ad313d84388957dfe0aa246a3d3 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d82c0ad313d84388957dfe0aa246a3d3 2024-12-10T06:17:03,974 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d82c0ad313d84388957dfe0aa246a3d3, entries=150, sequenceid=257, filesize=11.9 K 2024-12-10T06:17:03,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/aac6589e105f4857abc2822a1ab22001 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/aac6589e105f4857abc2822a1ab22001 2024-12-10T06:17:03,977 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/aac6589e105f4857abc2822a1ab22001, entries=150, sequenceid=257, filesize=11.9 K 2024-12-10T06:17:03,978 INFO [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=33.54 KB/34350 for 112cbd3eaf4337ea756788d925ebbcd9 in 1287ms, sequenceid=257, compaction requested=false 2024-12-10T06:17:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 112cbd3eaf4337ea756788d925ebbcd9: 2024-12-10T06:17:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-10T06:17:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-10T06:17:03,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-10T06:17:03,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5110 sec 2024-12-10T06:17:03,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.5150 sec 2024-12-10T06:17:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T06:17:05,646 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2888 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2897 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2862 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2888 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2900 2024-12-10T06:17:05,646 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T06:17:05,646 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:17:05,646 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:17:05,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:05,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:05,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:05,647 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T06:17:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T06:17:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T06:17:05,650 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811425650"}]},"ts":"1733811425650"} 2024-12-10T06:17:05,651 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T06:17:05,651 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T06:17:05,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T06:17:05,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, UNASSIGN}] 2024-12-10T06:17:05,653 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, UNASSIGN 2024-12-10T06:17:05,654 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=112cbd3eaf4337ea756788d925ebbcd9, regionState=CLOSING, regionLocation=c42c682b34f7,43983,1733811247584 2024-12-10T06:17:05,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, UNASSIGN because future has completed 2024-12-10T06:17:05,655 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:17:05,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584}] 2024-12-10T06:17:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T06:17:05,807 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:05,807 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 112cbd3eaf4337ea756788d925ebbcd9, disabling compactions & flushes 2024-12-10T06:17:05,808 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. after waiting 0 ms 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:05,808 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 112cbd3eaf4337ea756788d925ebbcd9 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=A 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=B 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 112cbd3eaf4337ea756788d925ebbcd9, store=C 2024-12-10T06:17:05,808 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T06:17:05,813 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e29a36f114144acd9a56247df8fdf67f_112cbd3eaf4337ea756788d925ebbcd9 is 50, key is test_row_0/A:col10/1733811423586/Put/seqid=0 2024-12-10T06:17:05,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742431_1607 (size=9914) 2024-12-10T06:17:05,916 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T06:17:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T06:17:06,216 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:17:06,219 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e29a36f114144acd9a56247df8fdf67f_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e29a36f114144acd9a56247df8fdf67f_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:06,220 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/0b02e1a975cf41e8aeb56d468e7e91d0, store: [table=TestAcidGuarantees family=A region=112cbd3eaf4337ea756788d925ebbcd9] 2024-12-10T06:17:06,220 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/0b02e1a975cf41e8aeb56d468e7e91d0 is 175, key is test_row_0/A:col10/1733811423586/Put/seqid=0 2024-12-10T06:17:06,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742432_1608 (size=22561) 2024-12-10T06:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T06:17:06,624 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/0b02e1a975cf41e8aeb56d468e7e91d0 2024-12-10T06:17:06,629 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8c64d7cf99564e398319478abc685b5e is 50, key is test_row_0/B:col10/1733811423586/Put/seqid=0 2024-12-10T06:17:06,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742433_1609 (size=9857) 2024-12-10T06:17:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T06:17:07,033 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8c64d7cf99564e398319478abc685b5e 2024-12-10T06:17:07,039 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/8a7eeb55cd814404b0b607837dffe5cf is 50, key is test_row_0/C:col10/1733811423586/Put/seqid=0 2024-12-10T06:17:07,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742434_1610 (size=9857) 2024-12-10T06:17:07,442 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/8a7eeb55cd814404b0b607837dffe5cf 2024-12-10T06:17:07,446 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/A/0b02e1a975cf41e8aeb56d468e7e91d0 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/0b02e1a975cf41e8aeb56d468e7e91d0 2024-12-10T06:17:07,448 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/0b02e1a975cf41e8aeb56d468e7e91d0, entries=100, sequenceid=268, filesize=22.0 K 2024-12-10T06:17:07,449 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/B/8c64d7cf99564e398319478abc685b5e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8c64d7cf99564e398319478abc685b5e 2024-12-10T06:17:07,451 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8c64d7cf99564e398319478abc685b5e, entries=100, sequenceid=268, filesize=9.6 K 2024-12-10T06:17:07,452 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/.tmp/C/8a7eeb55cd814404b0b607837dffe5cf as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/8a7eeb55cd814404b0b607837dffe5cf 2024-12-10T06:17:07,454 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/8a7eeb55cd814404b0b607837dffe5cf, entries=100, sequenceid=268, filesize=9.6 K 2024-12-10T06:17:07,455 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 112cbd3eaf4337ea756788d925ebbcd9 in 1647ms, sequenceid=268, compaction requested=true 2024-12-10T06:17:07,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/f9f182746e09421483fe3fc153516247, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/36139701d7354c2fb9dc82d25b541bb0, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/2c7c9e0f409b4549b276edd82d143a22, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/86544fe61f7d45fcb0864fd29140df71, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/80baeb738e0f4f2594100b0ce86cf2d6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/3fe1369b6ff542c78c69922b79cad843, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/b347f4755c524f2bb0e7412573f5ae2a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/d8269a6d9cf94e279a77669ed1c785e2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/cfe06308a2c349e19a24f9b2835c6145, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/fe69a34cc86241a0a2006a176049b129, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/72f472e072b045ad90e9650a2bb19512, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/ed47d912b57c412fb349375cf725b98b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6bb6609b08e3476d8e64f93f40e30ae7, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/4e616a377abe47d086b93c41f904bece, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/23ea383111a64f71b3ca06771575551a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/16298138185645e8a7ef30c4f1b842d1, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6f27598a44d94085b6c81d42f5a43018] to archive 2024-12-10T06:17:07,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:17:07,458 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/2c7c9e0f409b4549b276edd82d143a22 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/2c7c9e0f409b4549b276edd82d143a22 2024-12-10T06:17:07,458 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/36139701d7354c2fb9dc82d25b541bb0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/36139701d7354c2fb9dc82d25b541bb0 2024-12-10T06:17:07,458 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/f9f182746e09421483fe3fc153516247 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/f9f182746e09421483fe3fc153516247 2024-12-10T06:17:07,458 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/80baeb738e0f4f2594100b0ce86cf2d6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/80baeb738e0f4f2594100b0ce86cf2d6 2024-12-10T06:17:07,458 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/86544fe61f7d45fcb0864fd29140df71 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/86544fe61f7d45fcb0864fd29140df71 2024-12-10T06:17:07,458 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/3fe1369b6ff542c78c69922b79cad843 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/3fe1369b6ff542c78c69922b79cad843 2024-12-10T06:17:07,458 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/b347f4755c524f2bb0e7412573f5ae2a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/b347f4755c524f2bb0e7412573f5ae2a 2024-12-10T06:17:07,458 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/d8269a6d9cf94e279a77669ed1c785e2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/d8269a6d9cf94e279a77669ed1c785e2 2024-12-10T06:17:07,459 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/cfe06308a2c349e19a24f9b2835c6145 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/cfe06308a2c349e19a24f9b2835c6145 2024-12-10T06:17:07,459 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/4e616a377abe47d086b93c41f904bece to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/4e616a377abe47d086b93c41f904bece 2024-12-10T06:17:07,460 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/23ea383111a64f71b3ca06771575551a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/23ea383111a64f71b3ca06771575551a 2024-12-10T06:17:07,460 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/72f472e072b045ad90e9650a2bb19512 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/72f472e072b045ad90e9650a2bb19512 2024-12-10T06:17:07,460 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/fe69a34cc86241a0a2006a176049b129 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/fe69a34cc86241a0a2006a176049b129 2024-12-10T06:17:07,460 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/16298138185645e8a7ef30c4f1b842d1 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/16298138185645e8a7ef30c4f1b842d1 2024-12-10T06:17:07,460 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/ed47d912b57c412fb349375cf725b98b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/ed47d912b57c412fb349375cf725b98b 2024-12-10T06:17:07,460 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6bb6609b08e3476d8e64f93f40e30ae7 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6bb6609b08e3476d8e64f93f40e30ae7 2024-12-10T06:17:07,461 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6f27598a44d94085b6c81d42f5a43018 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/6f27598a44d94085b6c81d42f5a43018 2024-12-10T06:17:07,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ef27a7ede7a841ea8ed625a8c87e1758, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8d30f8d3c370427da6920fe6df1e0967, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8c949bfec22b43d9bcd180949142de3d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/74dea80187094b868c099998af738d96, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d44183030dc747f58ece836a31530697, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/67c39ab7ca8a44829d7f3efe689a0c9a, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/6743a0170868453d8220e041045c7c0b, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/50d0a366052a451e80b938a1567c17cc, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/bc842251cacb493aaedf8e9d39cb72cf, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d36b89874a88472e821ec03545b79d91, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8ea6c65c6cbe4c9cba02565530002833, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/7bd79898900c4f0e9e9863af0d6d280c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/33e18a68743d4d33b3e3df9169f24734, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/05939baeaf8848c8bb1ad7c221f93b67, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ff2206daa0ba480d80d3298d730d8372, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8661487b985d41be99dba236253abce4, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/028c501ac215417793bc9dfcbbc58730] to archive 2024-12-10T06:17:07,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:17:07,464 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8d30f8d3c370427da6920fe6df1e0967 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8d30f8d3c370427da6920fe6df1e0967 2024-12-10T06:17:07,465 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8c949bfec22b43d9bcd180949142de3d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8c949bfec22b43d9bcd180949142de3d 2024-12-10T06:17:07,465 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ef27a7ede7a841ea8ed625a8c87e1758 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ef27a7ede7a841ea8ed625a8c87e1758 2024-12-10T06:17:07,465 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d44183030dc747f58ece836a31530697 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d44183030dc747f58ece836a31530697 2024-12-10T06:17:07,465 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/6743a0170868453d8220e041045c7c0b to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/6743a0170868453d8220e041045c7c0b 2024-12-10T06:17:07,465 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/50d0a366052a451e80b938a1567c17cc to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/50d0a366052a451e80b938a1567c17cc 2024-12-10T06:17:07,465 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/67c39ab7ca8a44829d7f3efe689a0c9a to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/67c39ab7ca8a44829d7f3efe689a0c9a 2024-12-10T06:17:07,465 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/74dea80187094b868c099998af738d96 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/74dea80187094b868c099998af738d96 2024-12-10T06:17:07,466 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d36b89874a88472e821ec03545b79d91 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d36b89874a88472e821ec03545b79d91 2024-12-10T06:17:07,467 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8ea6c65c6cbe4c9cba02565530002833 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8ea6c65c6cbe4c9cba02565530002833 2024-12-10T06:17:07,467 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/33e18a68743d4d33b3e3df9169f24734 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/33e18a68743d4d33b3e3df9169f24734 2024-12-10T06:17:07,467 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/bc842251cacb493aaedf8e9d39cb72cf to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/bc842251cacb493aaedf8e9d39cb72cf 2024-12-10T06:17:07,467 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/05939baeaf8848c8bb1ad7c221f93b67 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/05939baeaf8848c8bb1ad7c221f93b67 2024-12-10T06:17:07,467 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/7bd79898900c4f0e9e9863af0d6d280c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/7bd79898900c4f0e9e9863af0d6d280c 2024-12-10T06:17:07,467 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ff2206daa0ba480d80d3298d730d8372 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/ff2206daa0ba480d80d3298d730d8372 2024-12-10T06:17:07,467 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8661487b985d41be99dba236253abce4 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8661487b985d41be99dba236253abce4 2024-12-10T06:17:07,468 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/028c501ac215417793bc9dfcbbc58730 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/028c501ac215417793bc9dfcbbc58730 2024-12-10T06:17:07,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/021acd5765384bb7960fd8b45ee15865, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/4100529f0b14411c898818c24590913d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/f55abdcd1de54d8b9d6e6e8cecfa54f2, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/51fad87d1f0041fd8188362eb9c30773, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/54fe0662b3c44ec9b33e388ea147d660, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/7dbb486028c34dc9a8501ec08bccf08c, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/17e3ced6f59f4af497fb64dce9b471fa, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/dad987db94374ec18d788b3947c55692, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b449417ba1054c85bca6c2aac37607ef, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/d571ec2d346c45ca87c8ded03bc2cb71, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/bc335f2bd8134bf681913879f6d0414d, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/589ffbacc8a14a38ad00b4dd7aa58481, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/2909c4c9b250435199a2f448925f6cb8, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/92383256df3744b9bea7493743fb6b35, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/6f0e900d36f6402da10a7c11abcc01c6, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b638556318fe4e65b528aecfae802105, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/99e32b8bb4b84f9f843df805d8b2b23c] to archive 2024-12-10T06:17:07,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:17:07,471 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/4100529f0b14411c898818c24590913d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/4100529f0b14411c898818c24590913d 2024-12-10T06:17:07,471 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/021acd5765384bb7960fd8b45ee15865 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/021acd5765384bb7960fd8b45ee15865 2024-12-10T06:17:07,471 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/f55abdcd1de54d8b9d6e6e8cecfa54f2 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/f55abdcd1de54d8b9d6e6e8cecfa54f2 2024-12-10T06:17:07,471 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/51fad87d1f0041fd8188362eb9c30773 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/51fad87d1f0041fd8188362eb9c30773 2024-12-10T06:17:07,471 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/54fe0662b3c44ec9b33e388ea147d660 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/54fe0662b3c44ec9b33e388ea147d660 2024-12-10T06:17:07,471 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/7dbb486028c34dc9a8501ec08bccf08c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/7dbb486028c34dc9a8501ec08bccf08c 2024-12-10T06:17:07,471 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/dad987db94374ec18d788b3947c55692 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/dad987db94374ec18d788b3947c55692 2024-12-10T06:17:07,471 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/17e3ced6f59f4af497fb64dce9b471fa to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/17e3ced6f59f4af497fb64dce9b471fa 2024-12-10T06:17:07,472 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/bc335f2bd8134bf681913879f6d0414d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/bc335f2bd8134bf681913879f6d0414d 2024-12-10T06:17:07,473 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/589ffbacc8a14a38ad00b4dd7aa58481 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/589ffbacc8a14a38ad00b4dd7aa58481 2024-12-10T06:17:07,473 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/92383256df3744b9bea7493743fb6b35 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/92383256df3744b9bea7493743fb6b35 2024-12-10T06:17:07,473 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/6f0e900d36f6402da10a7c11abcc01c6 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/6f0e900d36f6402da10a7c11abcc01c6 2024-12-10T06:17:07,473 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b449417ba1054c85bca6c2aac37607ef to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b449417ba1054c85bca6c2aac37607ef 2024-12-10T06:17:07,473 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/d571ec2d346c45ca87c8ded03bc2cb71 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/d571ec2d346c45ca87c8ded03bc2cb71 2024-12-10T06:17:07,473 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/2909c4c9b250435199a2f448925f6cb8 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/2909c4c9b250435199a2f448925f6cb8 2024-12-10T06:17:07,473 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b638556318fe4e65b528aecfae802105 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/b638556318fe4e65b528aecfae802105 2024-12-10T06:17:07,474 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/99e32b8bb4b84f9f843df805d8b2b23c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/99e32b8bb4b84f9f843df805d8b2b23c 2024-12-10T06:17:07,477 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/recovered.edits/271.seqid, newMaxSeqId=271, maxSeqId=4 2024-12-10T06:17:07,478 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. 2024-12-10T06:17:07,478 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 112cbd3eaf4337ea756788d925ebbcd9: Waiting for close lock at 1733811425808Running coprocessor pre-close hooks at 1733811425808Disabling compacts and flushes for region at 1733811425808Disabling writes for close at 1733811425808Obtaining lock to block concurrent updates at 1733811425808Preparing flush snapshotting stores in 112cbd3eaf4337ea756788d925ebbcd9 at 1733811425808Finished memstore snapshotting TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733811425808Flushing stores of TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9. at 1733811425809 (+1 ms)Flushing 112cbd3eaf4337ea756788d925ebbcd9/A: creating writer at 1733811425809Flushing mob file 112cbd3eaf4337ea756788d925ebbcd9/A: appending metadata at 1733811425812 (+3 ms)Flushing mob file 112cbd3eaf4337ea756788d925ebbcd9/A: closing flushed file at 1733811425812Flushing 112cbd3eaf4337ea756788d925ebbcd9/A: appending metadata at 1733811426220 (+408 ms)Flushing 112cbd3eaf4337ea756788d925ebbcd9/A: closing flushed file at 1733811426220Flushing 112cbd3eaf4337ea756788d925ebbcd9/B: creating writer at 1733811426627 (+407 ms)Flushing 112cbd3eaf4337ea756788d925ebbcd9/B: appending metadata at 1733811426629 (+2 ms)Flushing 112cbd3eaf4337ea756788d925ebbcd9/B: closing flushed file at 1733811426629Flushing 112cbd3eaf4337ea756788d925ebbcd9/C: creating writer at 1733811427036 (+407 ms)Flushing 112cbd3eaf4337ea756788d925ebbcd9/C: appending metadata at 1733811427038 (+2 ms)Flushing 112cbd3eaf4337ea756788d925ebbcd9/C: closing flushed file at 1733811427038Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4240e: reopening flushed file at 1733811427445 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e527a3: reopening flushed file at 1733811427449 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d270274: reopening flushed file at 1733811427451 (+2 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 112cbd3eaf4337ea756788d925ebbcd9 in 1647ms, sequenceid=268, compaction requested=true at 1733811427455 (+4 ms)Writing region close event to WAL at 1733811427475 (+20 ms)Running coprocessor post-close hooks at 1733811427478 (+3 ms)Closed at 1733811427478 2024-12-10T06:17:07,479 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,480 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=112cbd3eaf4337ea756788d925ebbcd9, regionState=CLOSED 2024-12-10T06:17:07,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 because future has completed 2024-12-10T06:17:07,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-10T06:17:07,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 112cbd3eaf4337ea756788d925ebbcd9, server=c42c682b34f7,43983,1733811247584 in 1.8270 sec 2024-12-10T06:17:07,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-12-10T06:17:07,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=112cbd3eaf4337ea756788d925ebbcd9, UNASSIGN in 1.8300 sec 2024-12-10T06:17:07,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-10T06:17:07,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8340 sec 2024-12-10T06:17:07,487 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733811427487"}]},"ts":"1733811427487"} 2024-12-10T06:17:07,488 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T06:17:07,488 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T06:17:07,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8420 sec 2024-12-10T06:17:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T06:17:07,796 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:17:07,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T06:17:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:17:07,798 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:17:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-10T06:17:07,798 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:17:07,800 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,802 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C, FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/recovered.edits] 2024-12-10T06:17:07,804 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/0b02e1a975cf41e8aeb56d468e7e91d0 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/0b02e1a975cf41e8aeb56d468e7e91d0 2024-12-10T06:17:07,804 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/32a2fdedfa674eaca1ad7050620bc17c to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/32a2fdedfa674eaca1ad7050620bc17c 2024-12-10T06:17:07,804 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/31f61521b6d74f52b44db23a91eabbe7 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/A/31f61521b6d74f52b44db23a91eabbe7 2024-12-10T06:17:07,806 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/18a34146ce254501a3a2fd5e5394b02d to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/18a34146ce254501a3a2fd5e5394b02d 2024-12-10T06:17:07,806 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8c64d7cf99564e398319478abc685b5e to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/8c64d7cf99564e398319478abc685b5e 2024-12-10T06:17:07,806 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d82c0ad313d84388957dfe0aa246a3d3 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/B/d82c0ad313d84388957dfe0aa246a3d3 2024-12-10T06:17:07,808 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/8a7eeb55cd814404b0b607837dffe5cf to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/8a7eeb55cd814404b0b607837dffe5cf 2024-12-10T06:17:07,808 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/aac6589e105f4857abc2822a1ab22001 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/aac6589e105f4857abc2822a1ab22001 2024-12-10T06:17:07,808 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/850b043860c448069e28a4cbaf82a574 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/C/850b043860c448069e28a4cbaf82a574 2024-12-10T06:17:07,810 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/recovered.edits/271.seqid to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9/recovered.edits/271.seqid 2024-12-10T06:17:07,811 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/default/TestAcidGuarantees/112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,811 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T06:17:07,811 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T06:17:07,812 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T06:17:07,815 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105f6c331c75f943f0999a8c1e372cba32_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105f6c331c75f943f0999a8c1e372cba32_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,816 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210349f8559ef8849d888ac6f54761519b2_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210349f8559ef8849d888ac6f54761519b2_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,816 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121063299451649148728c309683c17fef19_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121063299451649148728c309683c17fef19_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,816 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210709d6fb8c7a2459a87e6ddcfdcf53573_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210709d6fb8c7a2459a87e6ddcfdcf53573_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,816 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210729d17b190a647b0a7fa912a216b8ba7_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210729d17b190a647b0a7fa912a216b8ba7_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,816 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210847a99b53e054425ab7bb7a8195a328e_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210847a99b53e054425ab7bb7a8195a328e_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,816 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210748feacdcad24fdc95257e05bab73708_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210748feacdcad24fdc95257e05bab73708_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,816 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210932e707909f24d58bc5a5ac517fab0e2_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210932e707909f24d58bc5a5ac517fab0e2_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,817 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210aba4eb5c2fe1416d8fdd3968ad518b18_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210aba4eb5c2fe1416d8fdd3968ad518b18_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,817 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b52c8ac8a5ca4467839972d8f1abc3a4_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b52c8ac8a5ca4467839972d8f1abc3a4_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,817 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210aa6d84c5277349ff8665ad9e1992fa89_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210aa6d84c5277349ff8665ad9e1992fa89_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,817 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e29a36f114144acd9a56247df8fdf67f_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e29a36f114144acd9a56247df8fdf67f_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,817 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a87c189696594b34b34d629f68cd8e2b_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a87c189696594b34b34d629f68cd8e2b_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,817 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cf37494638c54b8ca76832c59f6d1a88_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cf37494638c54b8ca76832c59f6d1a88_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,817 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b74dbf51044048d3aa698a144c2d7f06_112cbd3eaf4337ea756788d925ebbcd9 to hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b74dbf51044048d3aa698a144c2d7f06_112cbd3eaf4337ea756788d925ebbcd9 2024-12-10T06:17:07,818 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T06:17:07,819 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:17:07,821 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T06:17:07,823 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T06:17:07,824 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:17:07,824 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T06:17:07,824 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733811427824"}]},"ts":"9223372036854775807"} 2024-12-10T06:17:07,826 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T06:17:07,826 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 112cbd3eaf4337ea756788d925ebbcd9, NAME => 'TestAcidGuarantees,,1733811400304.112cbd3eaf4337ea756788d925ebbcd9.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T06:17:07,826 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T06:17:07,826 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733811427826"}]},"ts":"9223372036854775807"} 2024-12-10T06:17:07,827 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T06:17:07,827 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T06:17:07,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-12-10T06:17:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37415 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-10T06:17:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T06:17:07,906 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T06:17:07,916 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=255 (was 253) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=355 (was 351) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5863 (was 5958) 2024-12-10T06:17:07,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T06:17:07,917 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:17:07,917 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:17:07,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:07,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:07,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:17:07,917 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T06:17:07,917 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1388394555, stopped=false 2024-12-10T06:17:07,917 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c42c682b34f7,37415,1733811246834 2024-12-10T06:17:07,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T06:17:07,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T06:17:07,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:17:07,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:17:07,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T06:17:07,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:17:07,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:17:07,920 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:17:07,920 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:17:07,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:07,921 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c42c682b34f7,43983,1733811247584' ***** 2024-12-10T06:17:07,921 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T06:17:07,921 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T06:17:07,922 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T06:17:07,922 INFO [RS:0;c42c682b34f7:43983 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T06:17:07,922 INFO [RS:0;c42c682b34f7:43983 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T06:17:07,922 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(959): stopping server c42c682b34f7,43983,1733811247584 2024-12-10T06:17:07,922 INFO [RS:0;c42c682b34f7:43983 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T06:17:07,922 INFO [RS:0;c42c682b34f7:43983 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c42c682b34f7:43983. 2024-12-10T06:17:07,922 DEBUG [RS:0;c42c682b34f7:43983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:17:07,922 DEBUG [RS:0;c42c682b34f7:43983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:07,922 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T06:17:07,922 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T06:17:07,922 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T06:17:07,922 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T06:17:07,923 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T06:17:07,923 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T06:17:07,923 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T06:17:07,923 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T06:17:07,923 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T06:17:07,923 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T06:17:07,923 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T06:17:07,923 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T06:17:07,924 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-10T06:17:07,949 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/info/c1859ead24e44eecbc5f4ed15ecb631e is 95, key is TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5./info:/1733811279462/DeleteFamily/seqid=0 2024-12-10T06:17:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742435_1611 (size=6514) 2024-12-10T06:17:07,953 INFO [regionserver/c42c682b34f7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T06:17:08,123 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T06:17:08,324 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T06:17:08,353 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/info/c1859ead24e44eecbc5f4ed15ecb631e 2024-12-10T06:17:08,373 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/ns/51acb6fa008f4e52966a751beb745fdb is 93, key is TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5./ns:/1733811279404/DeleteFamily/seqid=0 2024-12-10T06:17:08,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742436_1612 (size=5906) 2024-12-10T06:17:08,524 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T06:17:08,535 INFO [regionserver/c42c682b34f7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-10T06:17:08,535 INFO [regionserver/c42c682b34f7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-10T06:17:08,724 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T06:17:08,776 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/ns/51acb6fa008f4e52966a751beb745fdb 2024-12-10T06:17:08,794 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/rep_barrier/0726cd70b11e458fbae802edf9abb479 is 102, key is TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5./rep_barrier:/1733811279404/DeleteFamily/seqid=0 2024-12-10T06:17:08,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742437_1613 (size=6025) 2024-12-10T06:17:08,924 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T06:17:09,125 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T06:17:09,197 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/rep_barrier/0726cd70b11e458fbae802edf9abb479 2024-12-10T06:17:09,215 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/table/323e6de24bb74ea39a5aa7044dfaa657 is 96, key is TestAcidGuarantees,,1733811251583.91cf7a8c24ad1cfeb344be5ce72aa8a5./table:/1733811279404/DeleteFamily/seqid=0 2024-12-10T06:17:09,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742438_1614 (size=5936) 2024-12-10T06:17:09,325 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T06:17:09,525 DEBUG [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T06:17:09,619 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/table/323e6de24bb74ea39a5aa7044dfaa657 2024-12-10T06:17:09,622 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/info/c1859ead24e44eecbc5f4ed15ecb631e as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/info/c1859ead24e44eecbc5f4ed15ecb631e 2024-12-10T06:17:09,625 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/info/c1859ead24e44eecbc5f4ed15ecb631e, entries=12, sequenceid=90, filesize=6.4 K 2024-12-10T06:17:09,626 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/ns/51acb6fa008f4e52966a751beb745fdb as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/ns/51acb6fa008f4e52966a751beb745fdb 2024-12-10T06:17:09,628 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/ns/51acb6fa008f4e52966a751beb745fdb, entries=8, sequenceid=90, filesize=5.8 K 2024-12-10T06:17:09,629 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/rep_barrier/0726cd70b11e458fbae802edf9abb479 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/rep_barrier/0726cd70b11e458fbae802edf9abb479 2024-12-10T06:17:09,631 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/rep_barrier/0726cd70b11e458fbae802edf9abb479, entries=6, sequenceid=90, filesize=5.9 K 2024-12-10T06:17:09,632 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/.tmp/table/323e6de24bb74ea39a5aa7044dfaa657 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/table/323e6de24bb74ea39a5aa7044dfaa657 2024-12-10T06:17:09,634 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/table/323e6de24bb74ea39a5aa7044dfaa657, entries=7, sequenceid=90, filesize=5.8 K 2024-12-10T06:17:09,635 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1712ms, sequenceid=90, compaction requested=false 2024-12-10T06:17:09,639 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-10T06:17:09,640 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T06:17:09,640 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T06:17:09,640 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733811427923Running coprocessor pre-close hooks at 1733811427923Disabling compacts and flushes for region at 1733811427923Disabling writes for close at 1733811427923Obtaining lock to block concurrent updates at 1733811427924 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733811427924Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733811427924Flushing stores of hbase:meta,,1.1588230740 at 1733811427925 (+1 ms)Flushing 1588230740/info: creating writer at 1733811427925Flushing 1588230740/info: appending metadata at 1733811427949 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733811427949Flushing 1588230740/ns: creating writer at 1733811428357 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733811428372 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733811428372Flushing 1588230740/rep_barrier: creating writer at 1733811428779 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733811428793 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733811428793Flushing 1588230740/table: creating writer at 1733811429200 (+407 ms)Flushing 1588230740/table: appending metadata at 1733811429214 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733811429214Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@88be9d9: reopening flushed file at 1733811429622 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3637108b: reopening flushed file at 1733811429625 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@229b3dde: reopening flushed file at 1733811429628 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fff09c: reopening flushed file at 1733811429631 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1712ms, sequenceid=90, compaction requested=false at 1733811429635 (+4 ms)Writing region close event to WAL at 1733811429637 (+2 ms)Running coprocessor post-close hooks at 1733811429639 (+2 ms)Closed at 1733811429640 (+1 ms) 2024-12-10T06:17:09,640 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T06:17:09,725 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(976): stopping server c42c682b34f7,43983,1733811247584; all regions closed. 2024-12-10T06:17:09,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741834_1010 (size=25344) 2024-12-10T06:17:09,731 DEBUG [RS:0;c42c682b34f7:43983 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/oldWALs 2024-12-10T06:17:09,732 INFO [RS:0;c42c682b34f7:43983 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c42c682b34f7%2C43983%2C1733811247584.meta:.meta(num 1733811251113) 2024-12-10T06:17:09,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741833_1009 (size=15273866) 2024-12-10T06:17:09,735 DEBUG [RS:0;c42c682b34f7:43983 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/oldWALs 2024-12-10T06:17:09,735 INFO [RS:0;c42c682b34f7:43983 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c42c682b34f7%2C43983%2C1733811247584:(num 1733811250709) 2024-12-10T06:17:09,735 DEBUG [RS:0;c42c682b34f7:43983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:17:09,735 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T06:17:09,735 INFO [RS:0;c42c682b34f7:43983 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T06:17:09,735 INFO [RS:0;c42c682b34f7:43983 {}] hbase.ChoreService(370): Chore service for: regionserver/c42c682b34f7:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-10T06:17:09,736 INFO [RS:0;c42c682b34f7:43983 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T06:17:09,736 INFO [regionserver/c42c682b34f7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T06:17:09,736 INFO [RS:0;c42c682b34f7:43983 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43983 2024-12-10T06:17:09,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c42c682b34f7,43983,1733811247584 2024-12-10T06:17:09,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T06:17:09,740 INFO [RS:0;c42c682b34f7:43983 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T06:17:09,742 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c42c682b34f7,43983,1733811247584] 2024-12-10T06:17:09,745 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c42c682b34f7,43983,1733811247584 already deleted, retry=false 2024-12-10T06:17:09,746 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c42c682b34f7,43983,1733811247584 expired; onlineServers=0 2024-12-10T06:17:09,746 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c42c682b34f7,37415,1733811246834' ***** 2024-12-10T06:17:09,746 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T06:17:09,746 INFO [M:0;c42c682b34f7:37415 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T06:17:09,746 INFO [M:0;c42c682b34f7:37415 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T06:17:09,746 DEBUG [M:0;c42c682b34f7:37415 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T06:17:09,746 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T06:17:09,746 DEBUG [M:0;c42c682b34f7:37415 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T06:17:09,746 DEBUG [master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.large.0-1733811250362 {}] cleaner.HFileCleaner(306): Exit Thread[master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.large.0-1733811250362,5,FailOnTimeoutGroup] 2024-12-10T06:17:09,746 DEBUG [master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.small.0-1733811250368 {}] cleaner.HFileCleaner(306): Exit Thread[master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.small.0-1733811250368,5,FailOnTimeoutGroup] 2024-12-10T06:17:09,746 INFO [M:0;c42c682b34f7:37415 {}] hbase.ChoreService(370): Chore service for: master/c42c682b34f7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T06:17:09,747 INFO [M:0;c42c682b34f7:37415 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T06:17:09,747 DEBUG [M:0;c42c682b34f7:37415 {}] master.HMaster(1795): Stopping service threads 2024-12-10T06:17:09,747 INFO [M:0;c42c682b34f7:37415 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T06:17:09,747 INFO [M:0;c42c682b34f7:37415 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T06:17:09,747 INFO [M:0;c42c682b34f7:37415 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T06:17:09,748 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T06:17:09,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T06:17:09,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:17:09,748 DEBUG [M:0;c42c682b34f7:37415 {}] zookeeper.ZKUtil(347): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T06:17:09,748 WARN [M:0;c42c682b34f7:37415 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T06:17:09,749 INFO [M:0;c42c682b34f7:37415 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/.lastflushedseqids 2024-12-10T06:17:09,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742439_1615 (size=72) 2024-12-10T06:17:09,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:17:09,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1019b01fe2c0001, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:17:09,843 INFO [RS:0;c42c682b34f7:43983 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T06:17:09,843 INFO [RS:0;c42c682b34f7:43983 {}] regionserver.HRegionServer(1031): Exiting; stopping=c42c682b34f7,43983,1733811247584; zookeeper connection closed. 2024-12-10T06:17:09,844 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7ac6224b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7ac6224b 2024-12-10T06:17:09,844 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-10T06:17:10,159 INFO [M:0;c42c682b34f7:37415 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T06:17:10,159 INFO [M:0;c42c682b34f7:37415 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T06:17:10,159 DEBUG [M:0;c42c682b34f7:37415 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T06:17:10,159 INFO [M:0;c42c682b34f7:37415 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:17:10,159 DEBUG [M:0;c42c682b34f7:37415 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:17:10,159 DEBUG [M:0;c42c682b34f7:37415 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T06:17:10,159 DEBUG [M:0;c42c682b34f7:37415 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:17:10,159 INFO [M:0;c42c682b34f7:37415 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=764.18 KB heapSize=934.88 KB 2024-12-10T06:17:10,177 DEBUG [M:0;c42c682b34f7:37415 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1554df56d8e74f5fa5d5b3d5265e7fdb is 82, key is hbase:meta,,1/info:regioninfo/1733811251201/Put/seqid=0 2024-12-10T06:17:10,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742440_1616 (size=5672) 2024-12-10T06:17:10,581 INFO [M:0;c42c682b34f7:37415 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2108 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1554df56d8e74f5fa5d5b3d5265e7fdb 2024-12-10T06:17:10,602 DEBUG [M:0;c42c682b34f7:37415 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3c70c9f2e9c84d529c79efd6a6e8d134 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x8B/proc:d/1733811403369/Put/seqid=0 2024-12-10T06:17:10,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742441_1617 (size=44312) 2024-12-10T06:17:11,006 INFO [M:0;c42c682b34f7:37415 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=763.63 KB at sequenceid=2108 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3c70c9f2e9c84d529c79efd6a6e8d134 2024-12-10T06:17:11,009 INFO [M:0;c42c682b34f7:37415 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3c70c9f2e9c84d529c79efd6a6e8d134 2024-12-10T06:17:11,027 DEBUG [M:0;c42c682b34f7:37415 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/49b629af1688428e9a8d1bc50a7bfc2c is 69, key is c42c682b34f7,43983,1733811247584/rs:state/1733811250404/Put/seqid=0 2024-12-10T06:17:11,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073742442_1618 (size=5156) 2024-12-10T06:17:11,434 INFO [M:0;c42c682b34f7:37415 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2108 (bloomFilter=true), to=hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/49b629af1688428e9a8d1bc50a7bfc2c 2024-12-10T06:17:11,438 DEBUG [M:0;c42c682b34f7:37415 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1554df56d8e74f5fa5d5b3d5265e7fdb as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1554df56d8e74f5fa5d5b3d5265e7fdb 2024-12-10T06:17:11,441 INFO [M:0;c42c682b34f7:37415 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1554df56d8e74f5fa5d5b3d5265e7fdb, entries=8, sequenceid=2108, filesize=5.5 K 2024-12-10T06:17:11,442 DEBUG [M:0;c42c682b34f7:37415 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3c70c9f2e9c84d529c79efd6a6e8d134 as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3c70c9f2e9c84d529c79efd6a6e8d134 2024-12-10T06:17:11,444 INFO [M:0;c42c682b34f7:37415 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3c70c9f2e9c84d529c79efd6a6e8d134 2024-12-10T06:17:11,444 INFO [M:0;c42c682b34f7:37415 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3c70c9f2e9c84d529c79efd6a6e8d134, entries=164, sequenceid=2108, filesize=43.3 K 2024-12-10T06:17:11,445 DEBUG [M:0;c42c682b34f7:37415 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/49b629af1688428e9a8d1bc50a7bfc2c as hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/49b629af1688428e9a8d1bc50a7bfc2c 2024-12-10T06:17:11,447 INFO [M:0;c42c682b34f7:37415 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40237/user/jenkins/test-data/dc45cf47-ba71-fac2-204a-e8bfe20030d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/49b629af1688428e9a8d1bc50a7bfc2c, entries=1, sequenceid=2108, filesize=5.0 K 2024-12-10T06:17:11,448 INFO [M:0;c42c682b34f7:37415 {}] regionserver.HRegion(3140): Finished flush of dataSize ~764.18 KB/782521, heapSize ~934.58 KB/957008, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1289ms, sequenceid=2108, compaction requested=false 2024-12-10T06:17:11,449 INFO [M:0;c42c682b34f7:37415 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:17:11,449 DEBUG [M:0;c42c682b34f7:37415 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733811430159Disabling compacts and flushes for region at 1733811430159Disabling writes for close at 1733811430159Obtaining lock to block concurrent updates at 1733811430159Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733811430159Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=782521, getHeapSize=957248, getOffHeapSize=0, getCellsCount=2297 at 1733811430159Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733811430160 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733811430160Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733811430177 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733811430177Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733811430584 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733811430601 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733811430601Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733811431009 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733811431026 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733811431026Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4db343: reopening flushed file at 1733811431437 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4013c018: reopening flushed file at 1733811431441 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@329e16bf: reopening flushed file at 1733811431444 (+3 ms)Finished flush of dataSize ~764.18 KB/782521, heapSize ~934.58 KB/957008, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1289ms, sequenceid=2108, compaction requested=false at 1733811431448 (+4 ms)Writing region close event to WAL at 1733811431449 (+1 ms)Closed at 1733811431449 2024-12-10T06:17:11,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35903 is added to blk_1073741830_1006 (size=920716) 2024-12-10T06:17:11,452 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T06:17:11,452 INFO [M:0;c42c682b34f7:37415 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T06:17:11,452 INFO [M:0;c42c682b34f7:37415 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37415 2024-12-10T06:17:11,453 INFO [M:0;c42c682b34f7:37415 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T06:17:11,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:17:11,554 INFO [M:0;c42c682b34f7:37415 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T06:17:11,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37415-0x1019b01fe2c0000, quorum=127.0.0.1:55003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:17:11,559 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T06:17:11,561 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T06:17:11,562 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T06:17:11,562 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T06:17:11,562 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/hadoop.log.dir/,STOPPED} 2024-12-10T06:17:11,565 WARN [BP-959695638-172.17.0.2-1733811243885 heartbeating to localhost/127.0.0.1:40237 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T06:17:11,565 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T06:17:11,565 WARN [BP-959695638-172.17.0.2-1733811243885 heartbeating to localhost/127.0.0.1:40237 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-959695638-172.17.0.2-1733811243885 (Datanode Uuid 1053313a-5eb1-45b0-8331-7fb75bfc6be1) service to localhost/127.0.0.1:40237 2024-12-10T06:17:11,565 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T06:17:11,567 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982/data/data1/current/BP-959695638-172.17.0.2-1733811243885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T06:17:11,567 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/cluster_93cc914d-671a-80c5-6927-4ded2b579982/data/data2/current/BP-959695638-172.17.0.2-1733811243885 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T06:17:11,567 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T06:17:11,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T06:17:11,575 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T06:17:11,575 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T06:17:11,575 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T06:17:11,575 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b442dee-abbb-9397-6da7-68994dc9368d/hadoop.log.dir/,STOPPED} 2024-12-10T06:17:11,590 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T06:17:11,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down